{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import io\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import time\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "snr_0  = h5py.File('../Data/VAD_Training_data_SNR_0.mat')\n",
    "snr_5  = h5py.File('../Data/VAD_Training_data_SNR_5.mat')\n",
    "snr_10 = h5py.File('../Data/VAD_Training_data_SNR_10.mat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Shape  (1094400, 40, 40)\n",
      "Labels Shape (1094400, 2)\n"
     ]
    }
   ],
   "source": [
    "data   = np.empty([0,40,40])\n",
    "labels = np.empty([0,2])\n",
    "\n",
    "data = np.r_[data, np.transpose(snr_0['trainingData'].value, axes=(2,0,1))]\n",
    "data = np.r_[data, np.transpose(snr_5['trainingData'].value, axes=(2,0,1))]\n",
    "data = np.r_[data, np.transpose(snr_10['trainingData'].value, axes=(2,0,1))]\n",
    "data = np.r_[data, np.transpose(snr_0['testingData'].value, axes=(2,0,1))]\n",
    "data = np.r_[data, np.transpose(snr_5['testingData'].value, axes=(2,0,1))]\n",
    "data = np.r_[data, np.transpose(snr_10['testingData'].value, axes=(2,0,1))]\n",
    "\n",
    "labels = np.r_[labels, np.transpose(snr_0['trainingLabels'].value)]\n",
    "labels = np.r_[labels, np.transpose(snr_5['trainingLabels'].value)]\n",
    "labels = np.r_[labels, np.transpose(snr_10['trainingLabels'].value)]\n",
    "labels = np.r_[labels, np.transpose(snr_0['testingLabels'].value)]\n",
    "labels = np.r_[labels, np.transpose(snr_5['testingLabels'].value)]\n",
    "labels = np.r_[labels, np.transpose(snr_10['testingLabels'].value)]\n",
    "\n",
    "print(\"Data Shape \", data.shape)\n",
    "print(\"Labels Shape\", labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K = 40\n",
    "L = 20\n",
    "M = 10\n",
    "N = 100\n",
    "nClasses = 2\n",
    "div = 10\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.name_scope(\"hyperparameters\"):\n",
    "    learning_rate = tf.placeholder(tf.float32)\n",
    "    keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope(\"inputs\"):\n",
    "    x = tf.placeholder(tf.float32,[None, 40, 40], \"x-input\")    \n",
    "    x_image = tf.reshape(x, [-1, 40, 40, 1], name=\"x-image\")    \n",
    "    Y_ = tf.placeholder(tf.float32, [None, nClasses], \"y-input\")    \n",
    "    \n",
    "with tf.name_scope(\"model\"):\n",
    "    W1 = tf.Variable(tf.truncated_normal([5,5,1,K], stddev=0.05))\n",
    "    B1 = tf.Variable(tf.ones([K])/div)\n",
    "    Y1 = tf.nn.relu(tf.nn.conv2d(x_image, W1, strides=[1,2,2,1], padding='SAME') + B1)\n",
    "    \n",
    "    W2 = tf.Variable(tf.truncated_normal([5,5,K,L], stddev=0.05))\n",
    "    B2 = tf.Variable(tf.ones([L])/div)\n",
    "    Y2 = tf.nn.relu(tf.nn.conv2d(Y1, W2, strides=[1,2,2,1], padding='SAME') + B2)\n",
    "    \n",
    "    W3 = tf.Variable(tf.truncated_normal([5,5,L,M], stddev=0.05))\n",
    "    B3 = tf.Variable(tf.ones([M])/div)\n",
    "    Y3 = tf.nn.relu(tf.nn.conv2d(Y2, W3, strides=[1,2,2,1], padding='SAME') + B3)\n",
    "    \n",
    "    YY = tf.reshape(Y3, shape=[-1, 5*5*M])   \n",
    "    \n",
    "    W4 = tf.Variable(tf.truncated_normal([5*5*M, N], stddev=0.05))\n",
    "    B4 = tf.Variable(tf.ones([N])/div)\n",
    "    Yf = tf.nn.relu(tf.matmul(YY, W4) + B4)\n",
    "    Y4 = tf.nn.dropout(Yf, keep_prob)\n",
    "    \n",
    "    W5 = tf.Variable(tf.truncated_normal([N, 2], stddev=0.05))\n",
    "    B5 = tf.Variable(tf.ones([2])/div)\n",
    "    Y  = tf.nn.softmax(tf.matmul(Y4, W5) + B5)\n",
    "    \n",
    "with tf.name_scope(\"loss\"):\n",
    "    cross_entropy = -tf.reduce_sum(Y_*tf.log(tf.clip_by_value(Y,1e-10,1.0)))\n",
    "    is_correct = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))\n",
    "    \n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    train_op  = optimizer.minimize(cross_entropy) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:  0 step:      0, Training Accuracy: 75.20, loss: 306.3640\n",
      "epoch:  0 step:  12800, Training Accuracy: 87.50, loss: 184.5056\n",
      "epoch:  0 step:  25600, Training Accuracy: 93.36, loss: 119.0420\n",
      "epoch:  0 step:  38400, Training Accuracy: 92.97, loss: 111.5587\n",
      "epoch:  0 step:  51200, Training Accuracy: 92.58, loss: 116.3568\n",
      "epoch:  0 step:  64000, Training Accuracy: 93.95, loss: 87.4597\n",
      "epoch:  0 step:  76800, Training Accuracy: 94.73, loss: 81.7672\n",
      "epoch:  0 step:  89600, Training Accuracy: 95.90, loss: 78.5267\n",
      "epoch:  0 step: 102400, Training Accuracy: 94.73, loss: 73.7936\n",
      "epoch:  0 step: 115200, Training Accuracy: 94.92, loss: 78.0443\n",
      "epoch:  0 step: 128000, Training Accuracy: 94.14, loss: 84.5261\n",
      "epoch:  0 step: 140800, Training Accuracy: 94.34, loss: 92.8842\n",
      "epoch:  0 step: 153600, Training Accuracy: 93.36, loss: 94.3452\n",
      "epoch:  0 step: 166400, Training Accuracy: 96.88, loss: 58.3215\n",
      "epoch:  0 step: 179200, Training Accuracy: 94.53, loss: 90.6597\n",
      "epoch:  0 step: 192000, Training Accuracy: 96.88, loss: 57.4515\n",
      "epoch:  0 step: 204800, Training Accuracy: 96.88, loss: 54.7268\n",
      "epoch:  0 step: 217600, Training Accuracy: 95.70, loss: 74.4344\n",
      "epoch:  0 step: 230400, Training Accuracy: 95.90, loss: 67.2272\n",
      "epoch:  0 step: 243200, Training Accuracy: 95.51, loss: 78.3549\n",
      "epoch:  0 step: 256000, Training Accuracy: 95.90, loss: 59.8343\n",
      "epoch:  0 step: 268800, Training Accuracy: 97.66, loss: 44.9890\n",
      "epoch:  0 step: 281600, Training Accuracy: 96.68, loss: 60.5486\n",
      "epoch:  0 step: 294400, Training Accuracy: 95.51, loss: 66.7357\n",
      "epoch:  0 step: 307200, Training Accuracy: 96.48, loss: 60.1793\n",
      "epoch:  0 step: 320000, Training Accuracy: 96.29, loss: 64.4169\n",
      "epoch:  0 step: 332800, Training Accuracy: 96.48, loss: 66.9375\n",
      "epoch:  0 step: 345600, Training Accuracy: 97.27, loss: 47.4238\n",
      "epoch:  0 step: 358400, Training Accuracy: 96.88, loss: 57.2904\n",
      "epoch:  0 step: 371200, Training Accuracy: 95.51, loss: 68.4144\n",
      "epoch:  0 step: 384000, Training Accuracy: 96.88, loss: 52.4822\n",
      "epoch:  0 step: 396800, Training Accuracy: 96.48, loss: 61.7956\n",
      "epoch:  0 step: 409600, Training Accuracy: 95.70, loss: 60.3069\n",
      "epoch:  0 step: 422400, Training Accuracy: 97.66, loss: 54.0080\n",
      "epoch:  0 step: 435200, Training Accuracy: 95.90, loss: 70.2769\n",
      "epoch:  0 step: 448000, Training Accuracy: 96.68, loss: 51.4386\n",
      "epoch:  0 step: 460800, Training Accuracy: 96.68, loss: 57.1295\n",
      "epoch:  0 step: 473600, Training Accuracy: 94.14, loss: 86.4579\n",
      "epoch:  0 step: 486400, Training Accuracy: 96.09, loss: 61.9755\n",
      "epoch:  0 step: 499200, Training Accuracy: 95.70, loss: 70.8359\n",
      "epoch:  0 step: 512000, Training Accuracy: 96.68, loss: 54.2394\n",
      "epoch:  0 step: 524800, Training Accuracy: 97.46, loss: 55.9992\n",
      "epoch:  0 step: 537600, Training Accuracy: 97.07, loss: 56.7290\n",
      "epoch:  0 step: 550400, Training Accuracy: 96.09, loss: 59.7856\n",
      "epoch:  0 step: 563200, Training Accuracy: 96.48, loss: 64.2553\n",
      "epoch:  0 step: 576000, Training Accuracy: 96.68, loss: 61.0782\n",
      "epoch:  0 step: 588800, Training Accuracy: 96.29, loss: 58.6422\n",
      "epoch:  0 step: 601600, Training Accuracy: 97.46, loss: 47.1924\n",
      "epoch:  0 step: 614400, Training Accuracy: 96.88, loss: 59.4466\n",
      "epoch:  0 step: 627200, Training Accuracy: 96.29, loss: 52.7514\n",
      "epoch:  0 step: 640000, Training Accuracy: 95.70, loss: 66.7686\n",
      "epoch:  0 step: 652800, Training Accuracy: 95.90, loss: 64.9227\n",
      "epoch:  0 step: 665600, Training Accuracy: 96.48, loss: 52.2834\n",
      "epoch:  0 step: 678400, Training Accuracy: 96.09, loss: 74.5675\n",
      "epoch:  0 step: 691200, Training Accuracy: 97.46, loss: 50.8846\n",
      "epoch:  0 step: 704000, Training Accuracy: 96.29, loss: 62.5293\n",
      "epoch:  0 step: 716800, Training Accuracy: 97.07, loss: 62.2156\n",
      "epoch:  0 step: 729600, Training Accuracy: 96.68, loss: 54.8613\n",
      "epoch:  0 step: 742400, Training Accuracy: 97.46, loss: 44.8965\n",
      "epoch:  0 step: 755200, Training Accuracy: 97.27, loss: 52.2168\n",
      "epoch:  0 step: 768000, Training Accuracy: 96.88, loss: 57.5468\n",
      "epoch:  0 step: 780800, Training Accuracy: 96.88, loss: 54.9757\n",
      "epoch:  0 step: 793600, Training Accuracy: 97.46, loss: 48.7957\n",
      "epoch:  0 step: 806400, Training Accuracy: 96.68, loss: 59.6567\n",
      "epoch:  0 step: 819200, Training Accuracy: 97.66, loss: 44.5745\n",
      "epoch:  0 step: 832000, Training Accuracy: 96.09, loss: 64.6653\n",
      "epoch:  0 step: 844800, Training Accuracy: 95.90, loss: 61.9978\n",
      "epoch:  0 step: 857600, Training Accuracy: 96.88, loss: 47.1208\n",
      "epoch:  0 step: 870400, Training Accuracy: 96.09, loss: 73.2707\n",
      "epoch:  0 step: 883200, Training Accuracy: 96.48, loss: 51.1145\n",
      "epoch:  0 step: 896000, Training Accuracy: 96.29, loss: 56.9812\n",
      "epoch:  0 step: 908800, Training Accuracy: 95.70, loss: 70.4221\n",
      "epoch:  0 step: 921600, Training Accuracy: 95.90, loss: 60.8577\n",
      "epoch:  0 step: 934400, Training Accuracy: 97.07, loss: 57.6392\n",
      "epoch:  0 step: 947200, Training Accuracy: 96.68, loss: 48.5503\n",
      "epoch:  0 step: 960000, Training Accuracy: 97.46, loss: 39.7019\n",
      "epoch:  0 step: 972800, Training Accuracy: 97.85, loss: 40.1642\n",
      "epoch:  0 step: 985600, Training Accuracy: 95.12, loss: 71.7149\n",
      "epoch:  0 step: 998400, Training Accuracy: 97.66, loss: 47.4676\n",
      "epoch:  0 step: 1011200, Training Accuracy: 96.88, loss: 52.5598\n",
      "epoch:  0 step: 1024000, Training Accuracy: 96.68, loss: 58.7255\n",
      "epoch:  0 step: 1036800, Training Accuracy: 95.90, loss: 67.1014\n",
      "epoch:  0 step: 1049600, Training Accuracy: 97.46, loss: 38.9614\n",
      "epoch:  0 step: 1062400, Training Accuracy: 96.68, loss: 47.7788\n",
      "epoch:  0 step: 1075200, Training Accuracy: 95.51, loss: 60.5708\n",
      "epoch:  0 step: 1088000, Training Accuracy: 97.46, loss: 48.6312\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  1 step:      0, Training Accuracy: 97.66, loss: 44.0886\n",
      "epoch:  1 step:  12800, Training Accuracy: 97.46, loss: 47.9406\n",
      "epoch:  1 step:  25600, Training Accuracy: 97.27, loss: 48.4131\n",
      "epoch:  1 step:  38400, Training Accuracy: 96.88, loss: 55.0188\n",
      "epoch:  1 step:  51200, Training Accuracy: 97.66, loss: 42.7192\n",
      "epoch:  1 step:  64000, Training Accuracy: 97.07, loss: 49.0751\n",
      "epoch:  1 step:  76800, Training Accuracy: 96.88, loss: 61.5671\n",
      "epoch:  1 step:  89600, Training Accuracy: 97.27, loss: 54.5020\n",
      "epoch:  1 step: 102400, Training Accuracy: 96.48, loss: 59.2701\n",
      "epoch:  1 step: 115200, Training Accuracy: 96.29, loss: 44.9698\n",
      "epoch:  1 step: 128000, Training Accuracy: 97.85, loss: 41.9552\n",
      "epoch:  1 step: 140800, Training Accuracy: 96.88, loss: 48.3172\n",
      "epoch:  1 step: 153600, Training Accuracy: 96.09, loss: 64.7764\n",
      "epoch:  1 step: 166400, Training Accuracy: 97.27, loss: 50.6403\n",
      "epoch:  1 step: 179200, Training Accuracy: 97.07, loss: 46.0762\n",
      "epoch:  1 step: 192000, Training Accuracy: 98.44, loss: 29.3409\n",
      "epoch:  1 step: 204800, Training Accuracy: 96.48, loss: 64.9511\n",
      "epoch:  1 step: 217600, Training Accuracy: 97.85, loss: 38.1364\n",
      "epoch:  1 step: 230400, Training Accuracy: 97.46, loss: 43.7598\n",
      "epoch:  1 step: 243200, Training Accuracy: 96.88, loss: 57.3445\n",
      "epoch:  1 step: 256000, Training Accuracy: 98.05, loss: 45.5640\n",
      "epoch:  1 step: 268800, Training Accuracy: 97.85, loss: 46.0810\n",
      "epoch:  1 step: 281600, Training Accuracy: 97.46, loss: 48.9601\n",
      "epoch:  1 step: 294400, Training Accuracy: 97.85, loss: 37.5046\n",
      "epoch:  1 step: 307200, Training Accuracy: 96.88, loss: 51.0553\n",
      "epoch:  1 step: 320000, Training Accuracy: 97.07, loss: 54.9098\n",
      "epoch:  1 step: 332800, Training Accuracy: 98.05, loss: 37.4196\n",
      "epoch:  1 step: 345600, Training Accuracy: 96.68, loss: 62.9010\n",
      "epoch:  1 step: 358400, Training Accuracy: 96.68, loss: 54.5238\n",
      "epoch:  1 step: 371200, Training Accuracy: 96.29, loss: 64.8820\n",
      "epoch:  1 step: 384000, Training Accuracy: 97.27, loss: 40.3172\n",
      "epoch:  1 step: 396800, Training Accuracy: 97.46, loss: 42.5271\n",
      "epoch:  1 step: 409600, Training Accuracy: 98.44, loss: 36.3638\n",
      "epoch:  1 step: 422400, Training Accuracy: 97.27, loss: 43.0124\n",
      "epoch:  1 step: 435200, Training Accuracy: 95.90, loss: 64.7867\n",
      "epoch:  1 step: 448000, Training Accuracy: 98.63, loss: 31.7671\n",
      "epoch:  1 step: 460800, Training Accuracy: 96.88, loss: 51.8627\n",
      "epoch:  1 step: 473600, Training Accuracy: 97.66, loss: 39.1412\n",
      "epoch:  1 step: 486400, Training Accuracy: 97.85, loss: 42.0333\n",
      "epoch:  1 step: 499200, Training Accuracy: 97.66, loss: 46.8265\n",
      "epoch:  1 step: 512000, Training Accuracy: 96.88, loss: 55.9265\n",
      "epoch:  1 step: 524800, Training Accuracy: 97.85, loss: 41.8338\n",
      "epoch:  1 step: 537600, Training Accuracy: 97.85, loss: 40.0193\n",
      "epoch:  1 step: 550400, Training Accuracy: 96.48, loss: 64.5958\n",
      "epoch:  1 step: 563200, Training Accuracy: 97.66, loss: 46.6195\n",
      "epoch:  1 step: 576000, Training Accuracy: 98.24, loss: 41.1032\n",
      "epoch:  1 step: 588800, Training Accuracy: 98.05, loss: 39.1878\n",
      "epoch:  1 step: 601600, Training Accuracy: 96.29, loss: 64.5744\n",
      "epoch:  1 step: 614400, Training Accuracy: 95.70, loss: 54.9638\n",
      "epoch:  1 step: 627200, Training Accuracy: 95.90, loss: 57.2343\n",
      "epoch:  1 step: 640000, Training Accuracy: 96.29, loss: 67.2090\n",
      "epoch:  1 step: 652800, Training Accuracy: 97.46, loss: 46.8536\n",
      "epoch:  1 step: 665600, Training Accuracy: 96.68, loss: 47.2672\n",
      "epoch:  1 step: 678400, Training Accuracy: 96.68, loss: 50.7639\n",
      "epoch:  1 step: 691200, Training Accuracy: 98.44, loss: 36.0520\n",
      "epoch:  1 step: 704000, Training Accuracy: 96.09, loss: 70.5090\n",
      "epoch:  1 step: 716800, Training Accuracy: 96.29, loss: 60.6476\n",
      "epoch:  1 step: 729600, Training Accuracy: 97.07, loss: 55.9803\n",
      "epoch:  1 step: 742400, Training Accuracy: 97.07, loss: 51.9696\n",
      "epoch:  1 step: 755200, Training Accuracy: 96.09, loss: 62.0173\n",
      "epoch:  1 step: 768000, Training Accuracy: 97.07, loss: 47.1739\n",
      "epoch:  1 step: 780800, Training Accuracy: 96.48, loss: 61.7205\n",
      "epoch:  1 step: 793600, Training Accuracy: 96.88, loss: 59.8542\n",
      "epoch:  1 step: 806400, Training Accuracy: 96.48, loss: 52.2337\n",
      "epoch:  1 step: 819200, Training Accuracy: 97.46, loss: 34.7146\n",
      "epoch:  1 step: 832000, Training Accuracy: 96.48, loss: 58.4780\n",
      "epoch:  1 step: 844800, Training Accuracy: 94.92, loss: 84.9540\n",
      "epoch:  1 step: 857600, Training Accuracy: 97.46, loss: 52.7910\n",
      "epoch:  1 step: 870400, Training Accuracy: 97.66, loss: 47.3880\n",
      "epoch:  1 step: 883200, Training Accuracy: 96.88, loss: 55.0914\n",
      "epoch:  1 step: 896000, Training Accuracy: 97.07, loss: 52.5719\n",
      "epoch:  1 step: 908800, Training Accuracy: 96.68, loss: 56.2360\n",
      "epoch:  1 step: 921600, Training Accuracy: 97.27, loss: 55.3741\n",
      "epoch:  1 step: 934400, Training Accuracy: 97.46, loss: 50.3287\n",
      "epoch:  1 step: 947200, Training Accuracy: 97.27, loss: 51.1740\n",
      "epoch:  1 step: 960000, Training Accuracy: 97.46, loss: 37.2236\n",
      "epoch:  1 step: 972800, Training Accuracy: 97.66, loss: 46.2876\n",
      "epoch:  1 step: 985600, Training Accuracy: 98.05, loss: 42.1061\n",
      "epoch:  1 step: 998400, Training Accuracy: 96.68, loss: 55.8550\n",
      "epoch:  1 step: 1011200, Training Accuracy: 96.88, loss: 62.1014\n",
      "epoch:  1 step: 1024000, Training Accuracy: 95.70, loss: 81.2403\n",
      "epoch:  1 step: 1036800, Training Accuracy: 97.27, loss: 48.2400\n",
      "epoch:  1 step: 1049600, Training Accuracy: 96.88, loss: 63.3577\n",
      "epoch:  1 step: 1062400, Training Accuracy: 97.27, loss: 39.5152\n",
      "epoch:  1 step: 1075200, Training Accuracy: 96.29, loss: 67.7305\n",
      "epoch:  1 step: 1088000, Training Accuracy: 97.07, loss: 50.7476\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  2 step:      0, Training Accuracy: 97.07, loss: 45.2993\n",
      "epoch:  2 step:  12800, Training Accuracy: 97.85, loss: 48.5794\n",
      "epoch:  2 step:  25600, Training Accuracy: 98.05, loss: 39.1745\n",
      "epoch:  2 step:  38400, Training Accuracy: 97.66, loss: 53.1075\n",
      "epoch:  2 step:  51200, Training Accuracy: 96.68, loss: 51.5882\n",
      "epoch:  2 step:  64000, Training Accuracy: 97.46, loss: 39.1204\n",
      "epoch:  2 step:  76800, Training Accuracy: 98.44, loss: 35.3435\n",
      "epoch:  2 step:  89600, Training Accuracy: 97.85, loss: 40.9974\n",
      "epoch:  2 step: 102400, Training Accuracy: 96.88, loss: 59.4060\n",
      "epoch:  2 step: 115200, Training Accuracy: 97.66, loss: 49.7521\n",
      "epoch:  2 step: 128000, Training Accuracy: 97.66, loss: 38.7349\n",
      "epoch:  2 step: 140800, Training Accuracy: 97.66, loss: 42.6587\n",
      "epoch:  2 step: 153600, Training Accuracy: 97.27, loss: 51.0470\n",
      "epoch:  2 step: 166400, Training Accuracy: 97.07, loss: 50.2916\n",
      "epoch:  2 step: 179200, Training Accuracy: 96.88, loss: 50.3609\n",
      "epoch:  2 step: 192000, Training Accuracy: 96.29, loss: 67.4363\n",
      "epoch:  2 step: 204800, Training Accuracy: 98.05, loss: 41.4974\n",
      "epoch:  2 step: 217600, Training Accuracy: 97.46, loss: 47.5112\n",
      "epoch:  2 step: 230400, Training Accuracy: 96.88, loss: 56.6370\n",
      "epoch:  2 step: 243200, Training Accuracy: 98.24, loss: 30.9511\n",
      "epoch:  2 step: 256000, Training Accuracy: 97.66, loss: 37.5386\n",
      "epoch:  2 step: 268800, Training Accuracy: 95.90, loss: 68.3443\n",
      "epoch:  2 step: 281600, Training Accuracy: 96.29, loss: 55.3353\n",
      "epoch:  2 step: 294400, Training Accuracy: 97.27, loss: 47.5813\n",
      "epoch:  2 step: 307200, Training Accuracy: 95.70, loss: 67.3396\n",
      "epoch:  2 step: 320000, Training Accuracy: 96.88, loss: 47.1771\n",
      "epoch:  2 step: 332800, Training Accuracy: 98.24, loss: 41.2817\n",
      "epoch:  2 step: 345600, Training Accuracy: 97.85, loss: 39.0612\n",
      "epoch:  2 step: 358400, Training Accuracy: 97.66, loss: 48.4295\n",
      "epoch:  2 step: 371200, Training Accuracy: 97.85, loss: 47.0665\n",
      "epoch:  2 step: 384000, Training Accuracy: 96.68, loss: 52.1908\n",
      "epoch:  2 step: 396800, Training Accuracy: 97.66, loss: 49.7881\n",
      "epoch:  2 step: 409600, Training Accuracy: 98.83, loss: 32.7111\n",
      "epoch:  2 step: 422400, Training Accuracy: 98.24, loss: 41.4638\n",
      "epoch:  2 step: 435200, Training Accuracy: 96.88, loss: 49.3729\n",
      "epoch:  2 step: 448000, Training Accuracy: 98.05, loss: 31.9614\n",
      "epoch:  2 step: 460800, Training Accuracy: 96.88, loss: 52.9469\n",
      "epoch:  2 step: 473600, Training Accuracy: 97.27, loss: 56.9539\n",
      "epoch:  2 step: 486400, Training Accuracy: 97.85, loss: 30.5014\n",
      "epoch:  2 step: 499200, Training Accuracy: 97.66, loss: 39.2401\n",
      "epoch:  2 step: 512000, Training Accuracy: 97.07, loss: 49.7006\n",
      "epoch:  2 step: 524800, Training Accuracy: 97.85, loss: 39.6541\n",
      "epoch:  2 step: 537600, Training Accuracy: 96.29, loss: 60.9167\n",
      "epoch:  2 step: 550400, Training Accuracy: 96.29, loss: 71.8501\n",
      "epoch:  2 step: 563200, Training Accuracy: 97.46, loss: 37.6186\n",
      "epoch:  2 step: 576000, Training Accuracy: 97.66, loss: 43.8598\n",
      "epoch:  2 step: 588800, Training Accuracy: 96.68, loss: 53.7780\n",
      "epoch:  2 step: 601600, Training Accuracy: 97.27, loss: 41.7697\n",
      "epoch:  2 step: 614400, Training Accuracy: 97.66, loss: 38.7231\n",
      "epoch:  2 step: 627200, Training Accuracy: 97.66, loss: 38.9277\n",
      "epoch:  2 step: 640000, Training Accuracy: 97.27, loss: 48.9076\n",
      "epoch:  2 step: 652800, Training Accuracy: 98.24, loss: 31.6935\n",
      "epoch:  2 step: 665600, Training Accuracy: 97.07, loss: 49.4638\n",
      "epoch:  2 step: 678400, Training Accuracy: 98.05, loss: 41.7023\n",
      "epoch:  2 step: 691200, Training Accuracy: 98.24, loss: 37.1096\n",
      "epoch:  2 step: 704000, Training Accuracy: 98.05, loss: 36.0342\n",
      "epoch:  2 step: 716800, Training Accuracy: 96.29, loss: 63.6272\n",
      "epoch:  2 step: 729600, Training Accuracy: 97.66, loss: 38.8550\n",
      "epoch:  2 step: 742400, Training Accuracy: 98.63, loss: 28.0893\n",
      "epoch:  2 step: 755200, Training Accuracy: 97.07, loss: 46.5356\n",
      "epoch:  2 step: 768000, Training Accuracy: 96.88, loss: 60.4712\n",
      "epoch:  2 step: 780800, Training Accuracy: 96.68, loss: 58.7363\n",
      "epoch:  2 step: 793600, Training Accuracy: 96.88, loss: 62.9814\n",
      "epoch:  2 step: 806400, Training Accuracy: 97.07, loss: 54.9931\n",
      "epoch:  2 step: 819200, Training Accuracy: 98.05, loss: 43.1475\n",
      "epoch:  2 step: 832000, Training Accuracy: 95.90, loss: 59.7666\n",
      "epoch:  2 step: 844800, Training Accuracy: 96.29, loss: 61.8785\n",
      "epoch:  2 step: 857600, Training Accuracy: 97.66, loss: 54.5186\n",
      "epoch:  2 step: 870400, Training Accuracy: 98.44, loss: 30.4779\n",
      "epoch:  2 step: 883200, Training Accuracy: 97.66, loss: 45.5774\n",
      "epoch:  2 step: 896000, Training Accuracy: 97.66, loss: 46.1008\n",
      "epoch:  2 step: 908800, Training Accuracy: 97.85, loss: 47.3852\n",
      "epoch:  2 step: 921600, Training Accuracy: 98.05, loss: 32.5602\n",
      "epoch:  2 step: 934400, Training Accuracy: 97.07, loss: 47.6344\n",
      "epoch:  2 step: 947200, Training Accuracy: 98.44, loss: 29.1253\n",
      "epoch:  2 step: 960000, Training Accuracy: 96.29, loss: 63.2310\n",
      "epoch:  2 step: 972800, Training Accuracy: 97.27, loss: 41.9839\n",
      "epoch:  2 step: 985600, Training Accuracy: 97.46, loss: 58.0560\n",
      "epoch:  2 step: 998400, Training Accuracy: 98.24, loss: 29.9159\n",
      "epoch:  2 step: 1011200, Training Accuracy: 98.05, loss: 39.1150\n",
      "epoch:  2 step: 1024000, Training Accuracy: 97.27, loss: 59.4232\n",
      "epoch:  2 step: 1036800, Training Accuracy: 97.07, loss: 48.4785\n",
      "epoch:  2 step: 1049600, Training Accuracy: 98.05, loss: 44.6553\n",
      "epoch:  2 step: 1062400, Training Accuracy: 97.46, loss: 56.5286\n",
      "epoch:  2 step: 1075200, Training Accuracy: 97.07, loss: 54.0612\n",
      "epoch:  2 step: 1088000, Training Accuracy: 97.07, loss: 46.8720\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  3 step:      0, Training Accuracy: 97.46, loss: 55.7088\n",
      "epoch:  3 step:  12800, Training Accuracy: 96.29, loss: 63.5283\n",
      "epoch:  3 step:  25600, Training Accuracy: 96.68, loss: 61.2649\n",
      "epoch:  3 step:  38400, Training Accuracy: 98.63, loss: 29.1419\n",
      "epoch:  3 step:  51200, Training Accuracy: 97.27, loss: 46.0888\n",
      "epoch:  3 step:  64000, Training Accuracy: 96.68, loss: 63.6741\n",
      "epoch:  3 step:  76800, Training Accuracy: 97.07, loss: 49.2495\n",
      "epoch:  3 step:  89600, Training Accuracy: 97.66, loss: 49.3665\n",
      "epoch:  3 step: 102400, Training Accuracy: 97.85, loss: 34.5550\n",
      "epoch:  3 step: 115200, Training Accuracy: 97.07, loss: 46.7782\n",
      "epoch:  3 step: 128000, Training Accuracy: 98.05, loss: 40.1560\n",
      "epoch:  3 step: 140800, Training Accuracy: 96.88, loss: 50.8519\n",
      "epoch:  3 step: 153600, Training Accuracy: 97.46, loss: 38.2836\n",
      "epoch:  3 step: 166400, Training Accuracy: 97.66, loss: 47.0162\n",
      "epoch:  3 step: 179200, Training Accuracy: 96.29, loss: 68.6479\n",
      "epoch:  3 step: 192000, Training Accuracy: 97.46, loss: 40.5043\n",
      "epoch:  3 step: 204800, Training Accuracy: 97.46, loss: 46.8172\n",
      "epoch:  3 step: 217600, Training Accuracy: 98.24, loss: 38.5431\n",
      "epoch:  3 step: 230400, Training Accuracy: 96.68, loss: 59.3134\n",
      "epoch:  3 step: 243200, Training Accuracy: 97.27, loss: 59.8596\n",
      "epoch:  3 step: 256000, Training Accuracy: 98.05, loss: 37.8045\n",
      "epoch:  3 step: 268800, Training Accuracy: 98.05, loss: 41.8875\n",
      "epoch:  3 step: 281600, Training Accuracy: 97.27, loss: 49.0505\n",
      "epoch:  3 step: 294400, Training Accuracy: 98.05, loss: 43.6306\n",
      "epoch:  3 step: 307200, Training Accuracy: 98.83, loss: 31.5044\n",
      "epoch:  3 step: 320000, Training Accuracy: 98.63, loss: 33.5499\n",
      "epoch:  3 step: 332800, Training Accuracy: 97.46, loss: 49.4981\n",
      "epoch:  3 step: 345600, Training Accuracy: 97.46, loss: 47.1449\n",
      "epoch:  3 step: 358400, Training Accuracy: 98.83, loss: 25.0395\n",
      "epoch:  3 step: 371200, Training Accuracy: 98.44, loss: 33.4396\n",
      "epoch:  3 step: 384000, Training Accuracy: 97.66, loss: 44.9886\n",
      "epoch:  3 step: 396800, Training Accuracy: 97.66, loss: 51.4731\n",
      "epoch:  3 step: 409600, Training Accuracy: 97.85, loss: 43.3356\n",
      "epoch:  3 step: 422400, Training Accuracy: 97.85, loss: 46.5707\n",
      "epoch:  3 step: 435200, Training Accuracy: 96.88, loss: 58.3122\n",
      "epoch:  3 step: 448000, Training Accuracy: 97.46, loss: 39.0972\n",
      "epoch:  3 step: 460800, Training Accuracy: 97.46, loss: 50.6289\n",
      "epoch:  3 step: 473600, Training Accuracy: 96.88, loss: 51.5681\n",
      "epoch:  3 step: 486400, Training Accuracy: 97.07, loss: 51.6156\n",
      "epoch:  3 step: 499200, Training Accuracy: 97.66, loss: 42.1136\n",
      "epoch:  3 step: 512000, Training Accuracy: 97.27, loss: 44.9396\n",
      "epoch:  3 step: 524800, Training Accuracy: 96.29, loss: 60.4191\n",
      "epoch:  3 step: 537600, Training Accuracy: 98.24, loss: 30.3191\n",
      "epoch:  3 step: 550400, Training Accuracy: 97.07, loss: 49.5571\n",
      "epoch:  3 step: 563200, Training Accuracy: 98.24, loss: 35.7207\n",
      "epoch:  3 step: 576000, Training Accuracy: 96.68, loss: 57.6679\n",
      "epoch:  3 step: 588800, Training Accuracy: 96.29, loss: 65.1385\n",
      "epoch:  3 step: 601600, Training Accuracy: 98.44, loss: 31.7630\n",
      "epoch:  3 step: 614400, Training Accuracy: 96.09, loss: 71.7937\n",
      "epoch:  3 step: 627200, Training Accuracy: 97.66, loss: 53.8968\n",
      "epoch:  3 step: 640000, Training Accuracy: 98.24, loss: 38.9184\n",
      "epoch:  3 step: 652800, Training Accuracy: 98.44, loss: 31.1680\n",
      "epoch:  3 step: 665600, Training Accuracy: 98.24, loss: 38.9251\n",
      "epoch:  3 step: 678400, Training Accuracy: 97.66, loss: 35.1526\n",
      "epoch:  3 step: 691200, Training Accuracy: 98.44, loss: 32.9662\n",
      "epoch:  3 step: 704000, Training Accuracy: 96.68, loss: 43.9762\n",
      "epoch:  3 step: 716800, Training Accuracy: 98.05, loss: 37.8348\n",
      "epoch:  3 step: 729600, Training Accuracy: 98.05, loss: 35.3817\n",
      "epoch:  3 step: 742400, Training Accuracy: 98.44, loss: 30.3941\n",
      "epoch:  3 step: 755200, Training Accuracy: 96.88, loss: 53.6266\n",
      "epoch:  3 step: 768000, Training Accuracy: 98.05, loss: 32.8960\n",
      "epoch:  3 step: 780800, Training Accuracy: 97.07, loss: 48.6781\n",
      "epoch:  3 step: 793600, Training Accuracy: 97.66, loss: 43.9457\n",
      "epoch:  3 step: 806400, Training Accuracy: 97.07, loss: 58.5313\n",
      "epoch:  3 step: 819200, Training Accuracy: 96.68, loss: 53.7194\n",
      "epoch:  3 step: 832000, Training Accuracy: 96.88, loss: 50.0319\n",
      "epoch:  3 step: 844800, Training Accuracy: 96.68, loss: 53.0265\n",
      "epoch:  3 step: 857600, Training Accuracy: 98.05, loss: 35.2178\n",
      "epoch:  3 step: 870400, Training Accuracy: 98.44, loss: 36.6592\n",
      "epoch:  3 step: 883200, Training Accuracy: 96.68, loss: 57.1493\n",
      "epoch:  3 step: 896000, Training Accuracy: 96.29, loss: 66.3259\n",
      "epoch:  3 step: 908800, Training Accuracy: 97.46, loss: 50.2877\n",
      "epoch:  3 step: 921600, Training Accuracy: 97.85, loss: 46.1391\n",
      "epoch:  3 step: 934400, Training Accuracy: 97.66, loss: 37.8056\n",
      "epoch:  3 step: 947200, Training Accuracy: 97.66, loss: 37.6283\n",
      "epoch:  3 step: 960000, Training Accuracy: 97.27, loss: 52.6378\n",
      "epoch:  3 step: 972800, Training Accuracy: 96.88, loss: 41.0181\n",
      "epoch:  3 step: 985600, Training Accuracy: 98.83, loss: 24.9089\n",
      "epoch:  3 step: 998400, Training Accuracy: 97.46, loss: 46.9564\n",
      "epoch:  3 step: 1011200, Training Accuracy: 97.07, loss: 57.5027\n",
      "epoch:  3 step: 1024000, Training Accuracy: 97.27, loss: 48.0334\n",
      "epoch:  3 step: 1036800, Training Accuracy: 97.27, loss: 53.9527\n",
      "epoch:  3 step: 1049600, Training Accuracy: 97.46, loss: 48.3597\n",
      "epoch:  3 step: 1062400, Training Accuracy: 97.07, loss: 61.1776\n",
      "epoch:  3 step: 1075200, Training Accuracy: 97.46, loss: 43.4012\n",
      "epoch:  3 step: 1088000, Training Accuracy: 97.27, loss: 50.7399\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  4 step:      0, Training Accuracy: 97.27, loss: 47.5568\n",
      "epoch:  4 step:  12800, Training Accuracy: 97.46, loss: 45.8833\n",
      "epoch:  4 step:  25600, Training Accuracy: 97.85, loss: 51.0713\n",
      "epoch:  4 step:  38400, Training Accuracy: 95.90, loss: 63.8618\n",
      "epoch:  4 step:  51200, Training Accuracy: 97.46, loss: 46.0940\n",
      "epoch:  4 step:  64000, Training Accuracy: 98.44, loss: 36.2817\n",
      "epoch:  4 step:  76800, Training Accuracy: 97.27, loss: 46.3905\n",
      "epoch:  4 step:  89600, Training Accuracy: 98.44, loss: 35.6880\n",
      "epoch:  4 step: 102400, Training Accuracy: 98.44, loss: 25.8479\n",
      "epoch:  4 step: 115200, Training Accuracy: 98.05, loss: 34.9164\n",
      "epoch:  4 step: 128000, Training Accuracy: 98.63, loss: 28.7426\n",
      "epoch:  4 step: 140800, Training Accuracy: 96.68, loss: 52.3514\n",
      "epoch:  4 step: 153600, Training Accuracy: 98.44, loss: 29.7245\n",
      "epoch:  4 step: 166400, Training Accuracy: 97.66, loss: 46.4193\n",
      "epoch:  4 step: 179200, Training Accuracy: 97.07, loss: 47.4531\n",
      "epoch:  4 step: 192000, Training Accuracy: 97.85, loss: 40.0042\n",
      "epoch:  4 step: 204800, Training Accuracy: 98.05, loss: 41.8638\n",
      "epoch:  4 step: 217600, Training Accuracy: 98.44, loss: 26.0886\n",
      "epoch:  4 step: 230400, Training Accuracy: 97.66, loss: 44.2026\n",
      "epoch:  4 step: 243200, Training Accuracy: 98.05, loss: 39.8340\n",
      "epoch:  4 step: 256000, Training Accuracy: 97.46, loss: 38.2522\n",
      "epoch:  4 step: 268800, Training Accuracy: 98.24, loss: 33.2155\n",
      "epoch:  4 step: 281600, Training Accuracy: 97.46, loss: 36.9200\n",
      "epoch:  4 step: 294400, Training Accuracy: 97.46, loss: 43.8553\n",
      "epoch:  4 step: 307200, Training Accuracy: 98.83, loss: 23.1924\n",
      "epoch:  4 step: 320000, Training Accuracy: 97.66, loss: 45.1271\n",
      "epoch:  4 step: 332800, Training Accuracy: 97.07, loss: 47.2792\n",
      "epoch:  4 step: 345600, Training Accuracy: 97.46, loss: 46.2253\n",
      "epoch:  4 step: 358400, Training Accuracy: 96.68, loss: 49.1447\n",
      "epoch:  4 step: 371200, Training Accuracy: 96.88, loss: 53.2301\n",
      "epoch:  4 step: 384000, Training Accuracy: 98.44, loss: 35.0887\n",
      "epoch:  4 step: 396800, Training Accuracy: 96.29, loss: 62.5401\n",
      "epoch:  4 step: 409600, Training Accuracy: 97.46, loss: 49.3993\n",
      "epoch:  4 step: 422400, Training Accuracy: 96.88, loss: 51.9998\n",
      "epoch:  4 step: 435200, Training Accuracy: 98.24, loss: 29.2491\n",
      "epoch:  4 step: 448000, Training Accuracy: 98.05, loss: 39.5697\n",
      "epoch:  4 step: 460800, Training Accuracy: 97.85, loss: 31.7048\n",
      "epoch:  4 step: 473600, Training Accuracy: 96.48, loss: 65.7409\n",
      "epoch:  4 step: 486400, Training Accuracy: 97.46, loss: 49.3699\n",
      "epoch:  4 step: 499200, Training Accuracy: 97.46, loss: 50.5369\n",
      "epoch:  4 step: 512000, Training Accuracy: 98.24, loss: 35.0731\n",
      "epoch:  4 step: 524800, Training Accuracy: 98.63, loss: 29.4781\n",
      "epoch:  4 step: 537600, Training Accuracy: 98.44, loss: 37.7278\n",
      "epoch:  4 step: 550400, Training Accuracy: 97.07, loss: 60.3308\n",
      "epoch:  4 step: 563200, Training Accuracy: 98.24, loss: 37.1146\n",
      "epoch:  4 step: 576000, Training Accuracy: 97.27, loss: 45.0462\n",
      "epoch:  4 step: 588800, Training Accuracy: 96.09, loss: 60.4640\n",
      "epoch:  4 step: 601600, Training Accuracy: 96.48, loss: 59.0894\n",
      "epoch:  4 step: 614400, Training Accuracy: 96.48, loss: 63.1136\n",
      "epoch:  4 step: 627200, Training Accuracy: 97.27, loss: 47.2952\n",
      "epoch:  4 step: 640000, Training Accuracy: 98.05, loss: 41.8453\n",
      "epoch:  4 step: 652800, Training Accuracy: 97.85, loss: 41.3974\n",
      "epoch:  4 step: 665600, Training Accuracy: 96.88, loss: 60.2103\n",
      "epoch:  4 step: 678400, Training Accuracy: 97.46, loss: 43.3228\n",
      "epoch:  4 step: 691200, Training Accuracy: 98.44, loss: 26.2411\n",
      "epoch:  4 step: 704000, Training Accuracy: 96.68, loss: 57.4104\n",
      "epoch:  4 step: 716800, Training Accuracy: 98.63, loss: 34.1303\n",
      "epoch:  4 step: 729600, Training Accuracy: 98.24, loss: 27.6511\n",
      "epoch:  4 step: 742400, Training Accuracy: 98.44, loss: 34.3119\n",
      "epoch:  4 step: 755200, Training Accuracy: 96.88, loss: 62.6123\n",
      "epoch:  4 step: 768000, Training Accuracy: 98.05, loss: 39.5224\n",
      "epoch:  4 step: 780800, Training Accuracy: 97.46, loss: 44.5232\n",
      "epoch:  4 step: 793600, Training Accuracy: 98.24, loss: 39.7264\n",
      "epoch:  4 step: 806400, Training Accuracy: 97.66, loss: 43.3967\n",
      "epoch:  4 step: 819200, Training Accuracy: 98.44, loss: 36.6428\n",
      "epoch:  4 step: 832000, Training Accuracy: 99.02, loss: 21.6518\n",
      "epoch:  4 step: 844800, Training Accuracy: 98.05, loss: 32.8874\n",
      "epoch:  4 step: 857600, Training Accuracy: 96.48, loss: 53.2402\n",
      "epoch:  4 step: 870400, Training Accuracy: 97.85, loss: 40.4680\n",
      "epoch:  4 step: 883200, Training Accuracy: 97.46, loss: 50.3060\n",
      "epoch:  4 step: 896000, Training Accuracy: 98.05, loss: 41.1013\n",
      "epoch:  4 step: 908800, Training Accuracy: 97.46, loss: 53.9707\n",
      "epoch:  4 step: 921600, Training Accuracy: 97.07, loss: 49.5700\n",
      "epoch:  4 step: 934400, Training Accuracy: 97.07, loss: 56.1029\n",
      "epoch:  4 step: 947200, Training Accuracy: 97.66, loss: 47.0133\n",
      "epoch:  4 step: 960000, Training Accuracy: 97.27, loss: 42.0477\n",
      "epoch:  4 step: 972800, Training Accuracy: 97.27, loss: 53.1501\n",
      "epoch:  4 step: 985600, Training Accuracy: 97.07, loss: 50.3709\n",
      "epoch:  4 step: 998400, Training Accuracy: 97.46, loss: 52.5046\n",
      "epoch:  4 step: 1011200, Training Accuracy: 96.48, loss: 59.6873\n",
      "epoch:  4 step: 1024000, Training Accuracy: 98.63, loss: 30.2894\n",
      "epoch:  4 step: 1036800, Training Accuracy: 97.85, loss: 46.2602\n",
      "epoch:  4 step: 1049600, Training Accuracy: 98.44, loss: 37.5885\n",
      "epoch:  4 step: 1062400, Training Accuracy: 97.66, loss: 44.6446\n",
      "epoch:  4 step: 1075200, Training Accuracy: 96.68, loss: 49.0620\n",
      "epoch:  4 step: 1088000, Training Accuracy: 98.83, loss: 30.7255\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  5 step:      0, Training Accuracy: 97.07, loss: 52.0768\n",
      "epoch:  5 step:  12800, Training Accuracy: 97.46, loss: 49.0607\n",
      "epoch:  5 step:  25600, Training Accuracy: 97.66, loss: 38.8384\n",
      "epoch:  5 step:  38400, Training Accuracy: 97.46, loss: 39.9270\n",
      "epoch:  5 step:  51200, Training Accuracy: 98.44, loss: 30.4829\n",
      "epoch:  5 step:  64000, Training Accuracy: 98.05, loss: 39.3667\n",
      "epoch:  5 step:  76800, Training Accuracy: 97.85, loss: 38.0196\n",
      "epoch:  5 step:  89600, Training Accuracy: 97.66, loss: 45.2546\n",
      "epoch:  5 step: 102400, Training Accuracy: 98.83, loss: 28.3192\n",
      "epoch:  5 step: 115200, Training Accuracy: 97.27, loss: 45.1229\n",
      "epoch:  5 step: 128000, Training Accuracy: 96.29, loss: 65.9870\n",
      "epoch:  5 step: 140800, Training Accuracy: 97.66, loss: 44.1439\n",
      "epoch:  5 step: 153600, Training Accuracy: 98.24, loss: 36.3467\n",
      "epoch:  5 step: 166400, Training Accuracy: 98.24, loss: 45.0726\n",
      "epoch:  5 step: 179200, Training Accuracy: 97.27, loss: 55.3611\n",
      "epoch:  5 step: 192000, Training Accuracy: 98.24, loss: 35.6782\n",
      "epoch:  5 step: 204800, Training Accuracy: 96.48, loss: 54.3484\n",
      "epoch:  5 step: 217600, Training Accuracy: 97.46, loss: 41.8792\n",
      "epoch:  5 step: 230400, Training Accuracy: 98.05, loss: 39.8067\n",
      "epoch:  5 step: 243200, Training Accuracy: 97.07, loss: 48.7939\n",
      "epoch:  5 step: 256000, Training Accuracy: 97.85, loss: 47.2981\n",
      "epoch:  5 step: 268800, Training Accuracy: 97.27, loss: 51.7340\n",
      "epoch:  5 step: 281600, Training Accuracy: 97.66, loss: 36.0672\n",
      "epoch:  5 step: 294400, Training Accuracy: 98.24, loss: 32.2003\n",
      "epoch:  5 step: 307200, Training Accuracy: 97.85, loss: 40.0629\n",
      "epoch:  5 step: 320000, Training Accuracy: 97.85, loss: 42.2885\n",
      "epoch:  5 step: 332800, Training Accuracy: 98.63, loss: 36.1721\n",
      "epoch:  5 step: 345600, Training Accuracy: 99.41, loss: 23.5256\n",
      "epoch:  5 step: 358400, Training Accuracy: 97.66, loss: 51.3498\n",
      "epoch:  5 step: 371200, Training Accuracy: 98.83, loss: 27.7035\n",
      "epoch:  5 step: 384000, Training Accuracy: 96.88, loss: 56.7384\n",
      "epoch:  5 step: 396800, Training Accuracy: 96.88, loss: 53.5147\n",
      "epoch:  5 step: 409600, Training Accuracy: 97.66, loss: 56.8880\n",
      "epoch:  5 step: 422400, Training Accuracy: 97.66, loss: 40.3893\n",
      "epoch:  5 step: 435200, Training Accuracy: 96.88, loss: 52.4342\n",
      "epoch:  5 step: 448000, Training Accuracy: 96.68, loss: 58.0139\n",
      "epoch:  5 step: 460800, Training Accuracy: 96.29, loss: 72.8792\n",
      "epoch:  5 step: 473600, Training Accuracy: 98.83, loss: 31.8755\n",
      "epoch:  5 step: 486400, Training Accuracy: 97.46, loss: 45.8316\n",
      "epoch:  5 step: 499200, Training Accuracy: 98.83, loss: 31.8453\n",
      "epoch:  5 step: 512000, Training Accuracy: 96.88, loss: 56.8651\n",
      "epoch:  5 step: 524800, Training Accuracy: 97.66, loss: 37.2382\n",
      "epoch:  5 step: 537600, Training Accuracy: 97.27, loss: 65.8427\n",
      "epoch:  5 step: 550400, Training Accuracy: 97.07, loss: 59.5382\n",
      "epoch:  5 step: 563200, Training Accuracy: 98.44, loss: 36.0914\n",
      "epoch:  5 step: 576000, Training Accuracy: 97.27, loss: 49.9217\n",
      "epoch:  5 step: 588800, Training Accuracy: 97.66, loss: 43.4019\n",
      "epoch:  5 step: 601600, Training Accuracy: 97.46, loss: 51.0364\n",
      "epoch:  5 step: 614400, Training Accuracy: 97.66, loss: 46.4417\n",
      "epoch:  5 step: 627200, Training Accuracy: 98.05, loss: 37.3714\n",
      "epoch:  5 step: 640000, Training Accuracy: 97.27, loss: 48.8253\n",
      "epoch:  5 step: 652800, Training Accuracy: 99.22, loss: 22.2599\n",
      "epoch:  5 step: 665600, Training Accuracy: 98.05, loss: 32.2924\n",
      "epoch:  5 step: 678400, Training Accuracy: 98.63, loss: 28.4771\n",
      "epoch:  5 step: 691200, Training Accuracy: 97.85, loss: 47.3936\n",
      "epoch:  5 step: 704000, Training Accuracy: 98.05, loss: 35.5202\n",
      "epoch:  5 step: 716800, Training Accuracy: 97.66, loss: 36.0972\n",
      "epoch:  5 step: 729600, Training Accuracy: 98.63, loss: 30.2170\n",
      "epoch:  5 step: 742400, Training Accuracy: 98.24, loss: 37.7483\n",
      "epoch:  5 step: 755200, Training Accuracy: 98.24, loss: 30.6762\n",
      "epoch:  5 step: 768000, Training Accuracy: 97.46, loss: 46.6502\n",
      "epoch:  5 step: 780800, Training Accuracy: 98.44, loss: 31.5260\n",
      "epoch:  5 step: 793600, Training Accuracy: 97.66, loss: 43.4774\n",
      "epoch:  5 step: 806400, Training Accuracy: 98.05, loss: 39.1845\n",
      "epoch:  5 step: 819200, Training Accuracy: 98.05, loss: 35.1744\n",
      "epoch:  5 step: 832000, Training Accuracy: 98.05, loss: 38.0326\n",
      "epoch:  5 step: 844800, Training Accuracy: 98.24, loss: 39.6011\n",
      "epoch:  5 step: 857600, Training Accuracy: 97.46, loss: 48.7394\n",
      "epoch:  5 step: 870400, Training Accuracy: 97.46, loss: 44.5485\n",
      "epoch:  5 step: 883200, Training Accuracy: 97.46, loss: 43.4592\n",
      "epoch:  5 step: 896000, Training Accuracy: 99.41, loss: 15.3888\n",
      "epoch:  5 step: 908800, Training Accuracy: 96.88, loss: 51.7285\n",
      "epoch:  5 step: 921600, Training Accuracy: 96.29, loss: 60.4367\n",
      "epoch:  5 step: 934400, Training Accuracy: 97.66, loss: 43.0454\n",
      "epoch:  5 step: 947200, Training Accuracy: 97.66, loss: 38.8400\n",
      "epoch:  5 step: 960000, Training Accuracy: 97.66, loss: 41.8985\n",
      "epoch:  5 step: 972800, Training Accuracy: 97.07, loss: 50.3500\n",
      "epoch:  5 step: 985600, Training Accuracy: 97.27, loss: 51.5004\n",
      "epoch:  5 step: 998400, Training Accuracy: 97.66, loss: 43.9122\n",
      "epoch:  5 step: 1011200, Training Accuracy: 97.66, loss: 41.6880\n",
      "epoch:  5 step: 1024000, Training Accuracy: 99.02, loss: 23.3965\n",
      "epoch:  5 step: 1036800, Training Accuracy: 98.24, loss: 33.9345\n",
      "epoch:  5 step: 1049600, Training Accuracy: 97.46, loss: 54.6248\n",
      "epoch:  5 step: 1062400, Training Accuracy: 97.27, loss: 43.7230\n",
      "epoch:  5 step: 1075200, Training Accuracy: 97.07, loss: 49.9364\n",
      "epoch:  5 step: 1088000, Training Accuracy: 96.68, loss: 58.6704\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  6 step:      0, Training Accuracy: 97.27, loss: 45.7971\n",
      "epoch:  6 step:  12800, Training Accuracy: 97.27, loss: 42.9761\n",
      "epoch:  6 step:  25600, Training Accuracy: 97.46, loss: 49.2071\n",
      "epoch:  6 step:  38400, Training Accuracy: 98.63, loss: 19.0181\n",
      "epoch:  6 step:  51200, Training Accuracy: 98.63, loss: 32.5711\n",
      "epoch:  6 step:  64000, Training Accuracy: 98.63, loss: 27.2142\n",
      "epoch:  6 step:  76800, Training Accuracy: 97.66, loss: 44.9205\n",
      "epoch:  6 step:  89600, Training Accuracy: 96.29, loss: 66.1027\n",
      "epoch:  6 step: 102400, Training Accuracy: 98.83, loss: 25.3621\n",
      "epoch:  6 step: 115200, Training Accuracy: 97.27, loss: 52.5436\n",
      "epoch:  6 step: 128000, Training Accuracy: 97.27, loss: 51.5303\n",
      "epoch:  6 step: 140800, Training Accuracy: 97.85, loss: 38.0417\n",
      "epoch:  6 step: 153600, Training Accuracy: 98.24, loss: 28.2852\n",
      "epoch:  6 step: 166400, Training Accuracy: 97.85, loss: 45.6776\n",
      "epoch:  6 step: 179200, Training Accuracy: 97.27, loss: 43.8998\n",
      "epoch:  6 step: 192000, Training Accuracy: 98.44, loss: 31.1747\n",
      "epoch:  6 step: 204800, Training Accuracy: 98.05, loss: 45.0794\n",
      "epoch:  6 step: 217600, Training Accuracy: 97.27, loss: 54.1887\n",
      "epoch:  6 step: 230400, Training Accuracy: 97.66, loss: 41.4605\n",
      "epoch:  6 step: 243200, Training Accuracy: 97.27, loss: 47.1275\n",
      "epoch:  6 step: 256000, Training Accuracy: 97.66, loss: 47.4480\n",
      "epoch:  6 step: 268800, Training Accuracy: 97.66, loss: 50.7009\n",
      "epoch:  6 step: 281600, Training Accuracy: 98.24, loss: 29.2323\n",
      "epoch:  6 step: 294400, Training Accuracy: 97.27, loss: 48.0005\n",
      "epoch:  6 step: 307200, Training Accuracy: 97.66, loss: 41.0549\n",
      "epoch:  6 step: 320000, Training Accuracy: 98.44, loss: 30.7260\n",
      "epoch:  6 step: 332800, Training Accuracy: 97.66, loss: 32.7479\n",
      "epoch:  6 step: 345600, Training Accuracy: 97.27, loss: 42.5424\n",
      "epoch:  6 step: 358400, Training Accuracy: 98.24, loss: 40.8082\n",
      "epoch:  6 step: 371200, Training Accuracy: 97.85, loss: 41.4953\n",
      "epoch:  6 step: 384000, Training Accuracy: 98.83, loss: 31.7115\n",
      "epoch:  6 step: 396800, Training Accuracy: 98.05, loss: 34.5356\n",
      "epoch:  6 step: 409600, Training Accuracy: 98.05, loss: 37.3940\n",
      "epoch:  6 step: 422400, Training Accuracy: 98.63, loss: 27.8088\n",
      "epoch:  6 step: 435200, Training Accuracy: 97.66, loss: 40.4726\n",
      "epoch:  6 step: 448000, Training Accuracy: 98.63, loss: 32.6575\n",
      "epoch:  6 step: 460800, Training Accuracy: 98.05, loss: 40.3518\n",
      "epoch:  6 step: 473600, Training Accuracy: 98.83, loss: 23.8905\n",
      "epoch:  6 step: 486400, Training Accuracy: 98.63, loss: 31.8191\n",
      "epoch:  6 step: 499200, Training Accuracy: 97.46, loss: 42.4669\n",
      "epoch:  6 step: 512000, Training Accuracy: 98.24, loss: 29.1257\n",
      "epoch:  6 step: 524800, Training Accuracy: 98.24, loss: 37.7452\n",
      "epoch:  6 step: 537600, Training Accuracy: 98.63, loss: 26.9378\n",
      "epoch:  6 step: 550400, Training Accuracy: 98.24, loss: 39.1429\n",
      "epoch:  6 step: 563200, Training Accuracy: 98.44, loss: 36.8696\n",
      "epoch:  6 step: 576000, Training Accuracy: 99.02, loss: 23.7647\n",
      "epoch:  6 step: 588800, Training Accuracy: 98.05, loss: 31.6514\n",
      "epoch:  6 step: 601600, Training Accuracy: 97.07, loss: 58.1017\n",
      "epoch:  6 step: 614400, Training Accuracy: 98.05, loss: 33.2315\n",
      "epoch:  6 step: 627200, Training Accuracy: 98.83, loss: 32.7457\n",
      "epoch:  6 step: 640000, Training Accuracy: 98.24, loss: 32.5837\n",
      "epoch:  6 step: 652800, Training Accuracy: 97.46, loss: 42.4448\n",
      "epoch:  6 step: 665600, Training Accuracy: 97.85, loss: 33.4419\n",
      "epoch:  6 step: 678400, Training Accuracy: 98.05, loss: 34.4171\n",
      "epoch:  6 step: 691200, Training Accuracy: 98.44, loss: 35.6344\n",
      "epoch:  6 step: 704000, Training Accuracy: 97.46, loss: 52.6940\n",
      "epoch:  6 step: 716800, Training Accuracy: 98.05, loss: 43.4713\n",
      "epoch:  6 step: 729600, Training Accuracy: 97.46, loss: 41.2094\n",
      "epoch:  6 step: 742400, Training Accuracy: 97.85, loss: 39.8137\n",
      "epoch:  6 step: 755200, Training Accuracy: 97.66, loss: 45.8130\n",
      "epoch:  6 step: 768000, Training Accuracy: 99.61, loss: 16.6617\n",
      "epoch:  6 step: 780800, Training Accuracy: 96.68, loss: 52.9406\n",
      "epoch:  6 step: 793600, Training Accuracy: 97.07, loss: 44.6573\n",
      "epoch:  6 step: 806400, Training Accuracy: 98.24, loss: 35.4905\n",
      "epoch:  6 step: 819200, Training Accuracy: 97.66, loss: 32.5623\n",
      "epoch:  6 step: 832000, Training Accuracy: 97.85, loss: 41.9334\n",
      "epoch:  6 step: 844800, Training Accuracy: 97.85, loss: 54.1983\n",
      "epoch:  6 step: 857600, Training Accuracy: 96.88, loss: 59.0661\n",
      "epoch:  6 step: 870400, Training Accuracy: 98.24, loss: 43.0591\n",
      "epoch:  6 step: 883200, Training Accuracy: 97.46, loss: 45.1860\n",
      "epoch:  6 step: 896000, Training Accuracy: 98.44, loss: 34.2430\n",
      "epoch:  6 step: 908800, Training Accuracy: 98.05, loss: 33.5660\n",
      "epoch:  6 step: 921600, Training Accuracy: 96.48, loss: 63.9820\n",
      "epoch:  6 step: 934400, Training Accuracy: 98.05, loss: 32.8429\n",
      "epoch:  6 step: 947200, Training Accuracy: 96.68, loss: 60.2798\n",
      "epoch:  6 step: 960000, Training Accuracy: 97.46, loss: 51.7687\n",
      "epoch:  6 step: 972800, Training Accuracy: 97.27, loss: 48.9780\n",
      "epoch:  6 step: 985600, Training Accuracy: 97.27, loss: 53.2766\n",
      "epoch:  6 step: 998400, Training Accuracy: 98.05, loss: 37.3814\n",
      "epoch:  6 step: 1011200, Training Accuracy: 97.85, loss: 42.2136\n",
      "epoch:  6 step: 1024000, Training Accuracy: 97.27, loss: 45.5093\n",
      "epoch:  6 step: 1036800, Training Accuracy: 97.46, loss: 41.9624\n",
      "epoch:  6 step: 1049600, Training Accuracy: 97.46, loss: 44.3907\n",
      "epoch:  6 step: 1062400, Training Accuracy: 98.05, loss: 37.6946\n",
      "epoch:  6 step: 1075200, Training Accuracy: 97.85, loss: 39.4500\n",
      "epoch:  6 step: 1088000, Training Accuracy: 98.63, loss: 30.6757\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  7 step:      0, Training Accuracy: 98.83, loss: 27.1818\n",
      "epoch:  7 step:  12800, Training Accuracy: 99.02, loss: 19.8570\n",
      "epoch:  7 step:  25600, Training Accuracy: 98.05, loss: 36.2004\n",
      "epoch:  7 step:  38400, Training Accuracy: 98.63, loss: 34.6405\n",
      "epoch:  7 step:  51200, Training Accuracy: 98.05, loss: 39.6632\n",
      "epoch:  7 step:  64000, Training Accuracy: 98.44, loss: 27.3905\n",
      "epoch:  7 step:  76800, Training Accuracy: 97.66, loss: 42.9900\n",
      "epoch:  7 step:  89600, Training Accuracy: 98.05, loss: 33.5969\n",
      "epoch:  7 step: 102400, Training Accuracy: 98.24, loss: 35.0409\n",
      "epoch:  7 step: 115200, Training Accuracy: 98.44, loss: 32.1037\n",
      "epoch:  7 step: 128000, Training Accuracy: 97.66, loss: 41.8821\n",
      "epoch:  7 step: 140800, Training Accuracy: 97.46, loss: 46.7987\n",
      "epoch:  7 step: 153600, Training Accuracy: 97.66, loss: 50.1304\n",
      "epoch:  7 step: 166400, Training Accuracy: 98.05, loss: 41.2166\n",
      "epoch:  7 step: 179200, Training Accuracy: 98.24, loss: 39.1258\n",
      "epoch:  7 step: 192000, Training Accuracy: 97.27, loss: 52.0082\n",
      "epoch:  7 step: 204800, Training Accuracy: 98.05, loss: 45.8727\n",
      "epoch:  7 step: 217600, Training Accuracy: 98.24, loss: 34.3424\n",
      "epoch:  7 step: 230400, Training Accuracy: 98.24, loss: 35.0194\n",
      "epoch:  7 step: 243200, Training Accuracy: 98.24, loss: 29.4472\n",
      "epoch:  7 step: 256000, Training Accuracy: 97.85, loss: 41.5471\n",
      "epoch:  7 step: 268800, Training Accuracy: 98.44, loss: 26.9675\n",
      "epoch:  7 step: 281600, Training Accuracy: 98.05, loss: 35.5597\n",
      "epoch:  7 step: 294400, Training Accuracy: 97.85, loss: 42.5848\n",
      "epoch:  7 step: 307200, Training Accuracy: 98.05, loss: 30.4516\n",
      "epoch:  7 step: 320000, Training Accuracy: 97.27, loss: 48.6476\n",
      "epoch:  7 step: 332800, Training Accuracy: 97.27, loss: 47.3847\n",
      "epoch:  7 step: 345600, Training Accuracy: 97.85, loss: 38.5914\n",
      "epoch:  7 step: 358400, Training Accuracy: 96.88, loss: 51.3120\n",
      "epoch:  7 step: 371200, Training Accuracy: 97.27, loss: 56.5311\n",
      "epoch:  7 step: 384000, Training Accuracy: 98.44, loss: 32.8936\n",
      "epoch:  7 step: 396800, Training Accuracy: 98.24, loss: 30.4630\n",
      "epoch:  7 step: 409600, Training Accuracy: 97.46, loss: 49.4333\n",
      "epoch:  7 step: 422400, Training Accuracy: 98.05, loss: 34.0665\n",
      "epoch:  7 step: 435200, Training Accuracy: 97.85, loss: 35.6523\n",
      "epoch:  7 step: 448000, Training Accuracy: 97.27, loss: 51.4786\n",
      "epoch:  7 step: 460800, Training Accuracy: 97.46, loss: 45.1861\n",
      "epoch:  7 step: 473600, Training Accuracy: 98.05, loss: 44.4132\n",
      "epoch:  7 step: 486400, Training Accuracy: 97.66, loss: 39.0230\n",
      "epoch:  7 step: 499200, Training Accuracy: 97.46, loss: 44.9161\n",
      "epoch:  7 step: 512000, Training Accuracy: 98.05, loss: 35.1424\n",
      "epoch:  7 step: 524800, Training Accuracy: 97.85, loss: 37.9235\n",
      "epoch:  7 step: 537600, Training Accuracy: 98.63, loss: 33.9011\n",
      "epoch:  7 step: 550400, Training Accuracy: 97.27, loss: 43.4558\n",
      "epoch:  7 step: 563200, Training Accuracy: 97.66, loss: 37.6520\n",
      "epoch:  7 step: 576000, Training Accuracy: 96.88, loss: 50.5433\n",
      "epoch:  7 step: 588800, Training Accuracy: 98.63, loss: 30.7111\n",
      "epoch:  7 step: 601600, Training Accuracy: 97.46, loss: 45.6740\n",
      "epoch:  7 step: 614400, Training Accuracy: 98.05, loss: 31.6514\n",
      "epoch:  7 step: 627200, Training Accuracy: 97.66, loss: 35.1778\n",
      "epoch:  7 step: 640000, Training Accuracy: 99.22, loss: 17.0372\n",
      "epoch:  7 step: 652800, Training Accuracy: 98.05, loss: 40.4358\n",
      "epoch:  7 step: 665600, Training Accuracy: 99.02, loss: 26.9048\n",
      "epoch:  7 step: 678400, Training Accuracy: 97.66, loss: 42.4847\n",
      "epoch:  7 step: 691200, Training Accuracy: 96.68, loss: 50.0268\n",
      "epoch:  7 step: 704000, Training Accuracy: 98.05, loss: 35.2347\n",
      "epoch:  7 step: 716800, Training Accuracy: 98.83, loss: 28.7158\n",
      "epoch:  7 step: 729600, Training Accuracy: 96.29, loss: 58.2035\n",
      "epoch:  7 step: 742400, Training Accuracy: 97.27, loss: 47.1049\n",
      "epoch:  7 step: 755200, Training Accuracy: 98.24, loss: 32.5734\n",
      "epoch:  7 step: 768000, Training Accuracy: 97.85, loss: 48.4736\n",
      "epoch:  7 step: 780800, Training Accuracy: 97.66, loss: 35.4131\n",
      "epoch:  7 step: 793600, Training Accuracy: 98.24, loss: 39.8077\n",
      "epoch:  7 step: 806400, Training Accuracy: 98.24, loss: 33.8909\n",
      "epoch:  7 step: 819200, Training Accuracy: 97.46, loss: 41.9995\n",
      "epoch:  7 step: 832000, Training Accuracy: 98.83, loss: 24.5388\n",
      "epoch:  7 step: 844800, Training Accuracy: 98.05, loss: 36.6644\n",
      "epoch:  7 step: 857600, Training Accuracy: 98.44, loss: 28.7305\n",
      "epoch:  7 step: 870400, Training Accuracy: 98.24, loss: 26.0607\n",
      "epoch:  7 step: 883200, Training Accuracy: 97.85, loss: 42.9019\n",
      "epoch:  7 step: 896000, Training Accuracy: 97.66, loss: 46.2249\n",
      "epoch:  7 step: 908800, Training Accuracy: 97.85, loss: 36.3742\n",
      "epoch:  7 step: 921600, Training Accuracy: 97.85, loss: 42.3866\n",
      "epoch:  7 step: 934400, Training Accuracy: 98.05, loss: 42.8582\n",
      "epoch:  7 step: 947200, Training Accuracy: 97.27, loss: 42.1310\n",
      "epoch:  7 step: 960000, Training Accuracy: 97.46, loss: 46.9357\n",
      "epoch:  7 step: 972800, Training Accuracy: 98.63, loss: 30.1024\n",
      "epoch:  7 step: 985600, Training Accuracy: 98.63, loss: 30.7657\n",
      "epoch:  7 step: 998400, Training Accuracy: 97.46, loss: 37.9694\n",
      "epoch:  7 step: 1011200, Training Accuracy: 99.02, loss: 18.4787\n",
      "epoch:  7 step: 1024000, Training Accuracy: 98.83, loss: 22.9394\n",
      "epoch:  7 step: 1036800, Training Accuracy: 98.44, loss: 29.6641\n",
      "epoch:  7 step: 1049600, Training Accuracy: 98.44, loss: 34.1213\n",
      "epoch:  7 step: 1062400, Training Accuracy: 97.66, loss: 51.3166\n",
      "epoch:  7 step: 1075200, Training Accuracy: 97.85, loss: 42.8835\n",
      "epoch:  7 step: 1088000, Training Accuracy: 98.63, loss: 28.2417\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  8 step:      0, Training Accuracy: 97.46, loss: 43.8421\n",
      "epoch:  8 step:  12800, Training Accuracy: 97.07, loss: 49.2550\n",
      "epoch:  8 step:  25600, Training Accuracy: 98.63, loss: 32.2286\n",
      "epoch:  8 step:  38400, Training Accuracy: 98.83, loss: 27.3575\n",
      "epoch:  8 step:  51200, Training Accuracy: 96.68, loss: 67.5960\n",
      "epoch:  8 step:  64000, Training Accuracy: 98.44, loss: 29.6534\n",
      "epoch:  8 step:  76800, Training Accuracy: 96.68, loss: 56.8786\n",
      "epoch:  8 step:  89600, Training Accuracy: 98.83, loss: 29.8237\n",
      "epoch:  8 step: 102400, Training Accuracy: 97.27, loss: 53.9202\n",
      "epoch:  8 step: 115200, Training Accuracy: 97.85, loss: 38.0685\n",
      "epoch:  8 step: 128000, Training Accuracy: 98.63, loss: 30.1864\n",
      "epoch:  8 step: 140800, Training Accuracy: 96.88, loss: 51.4336\n",
      "epoch:  8 step: 153600, Training Accuracy: 97.46, loss: 48.3977\n",
      "epoch:  8 step: 166400, Training Accuracy: 97.85, loss: 43.8521\n",
      "epoch:  8 step: 179200, Training Accuracy: 98.44, loss: 27.9425\n",
      "epoch:  8 step: 192000, Training Accuracy: 98.24, loss: 32.5950\n",
      "epoch:  8 step: 204800, Training Accuracy: 98.05, loss: 43.4350\n",
      "epoch:  8 step: 217600, Training Accuracy: 96.48, loss: 63.8896\n",
      "epoch:  8 step: 230400, Training Accuracy: 96.88, loss: 55.4647\n",
      "epoch:  8 step: 243200, Training Accuracy: 98.05, loss: 42.4785\n",
      "epoch:  8 step: 256000, Training Accuracy: 99.02, loss: 21.3652\n",
      "epoch:  8 step: 268800, Training Accuracy: 98.44, loss: 31.8973\n",
      "epoch:  8 step: 281600, Training Accuracy: 98.05, loss: 31.0578\n",
      "epoch:  8 step: 294400, Training Accuracy: 98.44, loss: 35.5909\n",
      "epoch:  8 step: 307200, Training Accuracy: 97.66, loss: 40.5971\n",
      "epoch:  8 step: 320000, Training Accuracy: 97.07, loss: 53.1649\n",
      "epoch:  8 step: 332800, Training Accuracy: 99.41, loss: 15.8001\n",
      "epoch:  8 step: 345600, Training Accuracy: 98.24, loss: 38.8140\n",
      "epoch:  8 step: 358400, Training Accuracy: 97.85, loss: 37.8340\n",
      "epoch:  8 step: 371200, Training Accuracy: 97.07, loss: 48.5739\n",
      "epoch:  8 step: 384000, Training Accuracy: 97.85, loss: 42.4579\n",
      "epoch:  8 step: 396800, Training Accuracy: 97.27, loss: 48.7839\n",
      "epoch:  8 step: 409600, Training Accuracy: 98.44, loss: 26.3080\n",
      "epoch:  8 step: 422400, Training Accuracy: 99.02, loss: 19.8143\n",
      "epoch:  8 step: 435200, Training Accuracy: 97.07, loss: 58.1477\n",
      "epoch:  8 step: 448000, Training Accuracy: 97.27, loss: 50.5590\n",
      "epoch:  8 step: 460800, Training Accuracy: 97.66, loss: 49.2974\n",
      "epoch:  8 step: 473600, Training Accuracy: 95.90, loss: 67.1858\n",
      "epoch:  8 step: 486400, Training Accuracy: 97.66, loss: 45.7137\n",
      "epoch:  8 step: 499200, Training Accuracy: 96.48, loss: 65.6875\n",
      "epoch:  8 step: 512000, Training Accuracy: 97.85, loss: 36.4130\n",
      "epoch:  8 step: 524800, Training Accuracy: 97.27, loss: 42.5231\n",
      "epoch:  8 step: 537600, Training Accuracy: 98.05, loss: 37.3929\n",
      "epoch:  8 step: 550400, Training Accuracy: 98.05, loss: 35.9655\n",
      "epoch:  8 step: 563200, Training Accuracy: 97.46, loss: 48.2547\n",
      "epoch:  8 step: 576000, Training Accuracy: 97.66, loss: 38.2740\n",
      "epoch:  8 step: 588800, Training Accuracy: 97.85, loss: 34.4696\n",
      "epoch:  8 step: 601600, Training Accuracy: 98.05, loss: 38.3839\n",
      "epoch:  8 step: 614400, Training Accuracy: 97.27, loss: 43.9188\n",
      "epoch:  8 step: 627200, Training Accuracy: 97.07, loss: 52.2204\n",
      "epoch:  8 step: 640000, Training Accuracy: 97.27, loss: 51.2434\n",
      "epoch:  8 step: 652800, Training Accuracy: 97.85, loss: 45.0314\n",
      "epoch:  8 step: 665600, Training Accuracy: 99.02, loss: 23.4371\n",
      "epoch:  8 step: 678400, Training Accuracy: 98.44, loss: 32.3850\n",
      "epoch:  8 step: 691200, Training Accuracy: 98.44, loss: 26.4496\n",
      "epoch:  8 step: 704000, Training Accuracy: 97.46, loss: 44.5980\n",
      "epoch:  8 step: 716800, Training Accuracy: 97.66, loss: 47.2036\n",
      "epoch:  8 step: 729600, Training Accuracy: 97.07, loss: 50.7005\n",
      "epoch:  8 step: 742400, Training Accuracy: 98.44, loss: 30.1234\n",
      "epoch:  8 step: 755200, Training Accuracy: 97.27, loss: 44.0922\n",
      "epoch:  8 step: 768000, Training Accuracy: 95.51, loss: 65.8139\n",
      "epoch:  8 step: 780800, Training Accuracy: 97.66, loss: 40.0191\n",
      "epoch:  8 step: 793600, Training Accuracy: 97.85, loss: 48.5526\n",
      "epoch:  8 step: 806400, Training Accuracy: 97.66, loss: 45.6040\n",
      "epoch:  8 step: 819200, Training Accuracy: 97.46, loss: 41.9176\n",
      "epoch:  8 step: 832000, Training Accuracy: 97.85, loss: 44.8584\n",
      "epoch:  8 step: 844800, Training Accuracy: 99.41, loss: 17.6817\n",
      "epoch:  8 step: 857600, Training Accuracy: 98.44, loss: 30.4565\n",
      "epoch:  8 step: 870400, Training Accuracy: 97.66, loss: 41.0302\n",
      "epoch:  8 step: 883200, Training Accuracy: 98.05, loss: 32.5629\n",
      "epoch:  8 step: 896000, Training Accuracy: 98.24, loss: 34.7462\n",
      "epoch:  8 step: 908800, Training Accuracy: 97.46, loss: 39.4702\n",
      "epoch:  8 step: 921600, Training Accuracy: 99.02, loss: 20.9569\n",
      "epoch:  8 step: 934400, Training Accuracy: 97.27, loss: 40.4755\n",
      "epoch:  8 step: 947200, Training Accuracy: 97.85, loss: 34.9503\n",
      "epoch:  8 step: 960000, Training Accuracy: 97.66, loss: 35.0296\n",
      "epoch:  8 step: 972800, Training Accuracy: 98.05, loss: 38.6670\n",
      "epoch:  8 step: 985600, Training Accuracy: 98.63, loss: 27.4832\n",
      "epoch:  8 step: 998400, Training Accuracy: 97.85, loss: 36.8211\n",
      "epoch:  8 step: 1011200, Training Accuracy: 98.44, loss: 33.2332\n",
      "epoch:  8 step: 1024000, Training Accuracy: 98.24, loss: 32.1082\n",
      "epoch:  8 step: 1036800, Training Accuracy: 98.63, loss: 30.1119\n",
      "epoch:  8 step: 1049600, Training Accuracy: 97.27, loss: 46.9205\n",
      "epoch:  8 step: 1062400, Training Accuracy: 98.44, loss: 27.4426\n",
      "epoch:  8 step: 1075200, Training Accuracy: 98.24, loss: 28.0231\n",
      "epoch:  8 step: 1088000, Training Accuracy: 98.05, loss: 39.5712\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch:  9 step:      0, Training Accuracy: 98.63, loss: 30.9271\n",
      "epoch:  9 step:  12800, Training Accuracy: 98.24, loss: 32.7691\n",
      "epoch:  9 step:  25600, Training Accuracy: 98.05, loss: 31.9228\n",
      "epoch:  9 step:  38400, Training Accuracy: 98.05, loss: 35.6934\n",
      "epoch:  9 step:  51200, Training Accuracy: 98.24, loss: 40.8124\n",
      "epoch:  9 step:  64000, Training Accuracy: 98.83, loss: 27.4027\n",
      "epoch:  9 step:  76800, Training Accuracy: 97.46, loss: 52.0051\n",
      "epoch:  9 step:  89600, Training Accuracy: 98.44, loss: 27.1963\n",
      "epoch:  9 step: 102400, Training Accuracy: 98.05, loss: 38.2382\n",
      "epoch:  9 step: 115200, Training Accuracy: 99.02, loss: 28.8654\n",
      "epoch:  9 step: 128000, Training Accuracy: 98.24, loss: 37.9850\n",
      "epoch:  9 step: 140800, Training Accuracy: 98.24, loss: 26.5905\n",
      "epoch:  9 step: 153600, Training Accuracy: 97.66, loss: 45.3922\n",
      "epoch:  9 step: 166400, Training Accuracy: 97.85, loss: 39.4313\n",
      "epoch:  9 step: 179200, Training Accuracy: 97.85, loss: 39.4723\n",
      "epoch:  9 step: 192000, Training Accuracy: 97.85, loss: 37.3446\n",
      "epoch:  9 step: 204800, Training Accuracy: 97.66, loss: 46.4782\n",
      "epoch:  9 step: 217600, Training Accuracy: 98.05, loss: 32.4077\n",
      "epoch:  9 step: 230400, Training Accuracy: 98.05, loss: 38.1218\n",
      "epoch:  9 step: 243200, Training Accuracy: 98.83, loss: 24.0968\n",
      "epoch:  9 step: 256000, Training Accuracy: 97.85, loss: 32.5988\n",
      "epoch:  9 step: 268800, Training Accuracy: 98.44, loss: 31.2891\n",
      "epoch:  9 step: 281600, Training Accuracy: 97.85, loss: 34.8655\n",
      "epoch:  9 step: 294400, Training Accuracy: 96.29, loss: 62.0765\n",
      "epoch:  9 step: 307200, Training Accuracy: 98.63, loss: 35.4651\n",
      "epoch:  9 step: 320000, Training Accuracy: 97.66, loss: 38.1054\n",
      "epoch:  9 step: 332800, Training Accuracy: 98.24, loss: 30.6902\n",
      "epoch:  9 step: 345600, Training Accuracy: 98.63, loss: 34.5910\n",
      "epoch:  9 step: 358400, Training Accuracy: 96.88, loss: 53.5779\n",
      "epoch:  9 step: 371200, Training Accuracy: 97.27, loss: 49.4065\n",
      "epoch:  9 step: 384000, Training Accuracy: 98.44, loss: 31.0749\n",
      "epoch:  9 step: 396800, Training Accuracy: 98.83, loss: 25.4754\n",
      "epoch:  9 step: 409600, Training Accuracy: 98.63, loss: 27.9389\n",
      "epoch:  9 step: 422400, Training Accuracy: 99.02, loss: 28.3111\n",
      "epoch:  9 step: 435200, Training Accuracy: 98.05, loss: 39.0858\n",
      "epoch:  9 step: 448000, Training Accuracy: 98.24, loss: 42.6860\n",
      "epoch:  9 step: 460800, Training Accuracy: 97.66, loss: 48.5097\n",
      "epoch:  9 step: 473600, Training Accuracy: 97.85, loss: 41.8639\n",
      "epoch:  9 step: 486400, Training Accuracy: 97.66, loss: 33.8966\n",
      "epoch:  9 step: 499200, Training Accuracy: 99.02, loss: 26.5376\n",
      "epoch:  9 step: 512000, Training Accuracy: 96.88, loss: 52.7986\n",
      "epoch:  9 step: 524800, Training Accuracy: 96.88, loss: 49.2039\n",
      "epoch:  9 step: 537600, Training Accuracy: 98.44, loss: 37.3064\n",
      "epoch:  9 step: 550400, Training Accuracy: 97.27, loss: 43.7894\n",
      "epoch:  9 step: 563200, Training Accuracy: 97.66, loss: 42.7407\n",
      "epoch:  9 step: 576000, Training Accuracy: 97.46, loss: 39.5741\n",
      "epoch:  9 step: 588800, Training Accuracy: 98.44, loss: 33.1748\n",
      "epoch:  9 step: 601600, Training Accuracy: 97.66, loss: 38.4712\n",
      "epoch:  9 step: 614400, Training Accuracy: 98.63, loss: 29.6583\n",
      "epoch:  9 step: 627200, Training Accuracy: 98.24, loss: 31.4380\n",
      "epoch:  9 step: 640000, Training Accuracy: 98.05, loss: 42.0226\n",
      "epoch:  9 step: 652800, Training Accuracy: 96.68, loss: 55.4123\n",
      "epoch:  9 step: 665600, Training Accuracy: 96.88, loss: 56.8696\n",
      "epoch:  9 step: 678400, Training Accuracy: 98.05, loss: 40.2131\n",
      "epoch:  9 step: 691200, Training Accuracy: 98.05, loss: 45.5066\n",
      "epoch:  9 step: 704000, Training Accuracy: 97.46, loss: 40.1280\n",
      "epoch:  9 step: 716800, Training Accuracy: 98.83, loss: 35.4085\n",
      "epoch:  9 step: 729600, Training Accuracy: 98.83, loss: 24.8934\n",
      "epoch:  9 step: 742400, Training Accuracy: 98.83, loss: 25.7437\n",
      "epoch:  9 step: 755200, Training Accuracy: 98.44, loss: 29.7226\n",
      "epoch:  9 step: 768000, Training Accuracy: 97.27, loss: 48.9079\n",
      "epoch:  9 step: 780800, Training Accuracy: 97.46, loss: 47.9711\n",
      "epoch:  9 step: 793600, Training Accuracy: 97.27, loss: 45.2272\n",
      "epoch:  9 step: 806400, Training Accuracy: 98.24, loss: 34.0661\n",
      "epoch:  9 step: 819200, Training Accuracy: 98.83, loss: 29.2330\n",
      "epoch:  9 step: 832000, Training Accuracy: 98.83, loss: 23.3036\n",
      "epoch:  9 step: 844800, Training Accuracy: 98.24, loss: 32.1788\n",
      "epoch:  9 step: 857600, Training Accuracy: 98.83, loss: 25.2186\n",
      "epoch:  9 step: 870400, Training Accuracy: 97.27, loss: 44.0627\n",
      "epoch:  9 step: 883200, Training Accuracy: 98.05, loss: 45.0783\n",
      "epoch:  9 step: 896000, Training Accuracy: 97.85, loss: 40.0217\n",
      "epoch:  9 step: 908800, Training Accuracy: 98.24, loss: 35.5636\n",
      "epoch:  9 step: 921600, Training Accuracy: 98.63, loss: 32.6259\n",
      "epoch:  9 step: 934400, Training Accuracy: 97.85, loss: 42.1049\n",
      "epoch:  9 step: 947200, Training Accuracy: 98.05, loss: 32.8056\n",
      "epoch:  9 step: 960000, Training Accuracy: 98.24, loss: 38.0198\n",
      "epoch:  9 step: 972800, Training Accuracy: 97.66, loss: 45.0156\n",
      "epoch:  9 step: 985600, Training Accuracy: 98.05, loss: 34.9212\n",
      "epoch:  9 step: 998400, Training Accuracy: 97.27, loss: 41.5183\n",
      "epoch:  9 step: 1011200, Training Accuracy: 96.88, loss: 48.8050\n",
      "epoch:  9 step: 1024000, Training Accuracy: 98.44, loss: 29.2558\n",
      "epoch:  9 step: 1036800, Training Accuracy: 98.24, loss: 28.9201\n",
      "epoch:  9 step: 1049600, Training Accuracy: 98.83, loss: 27.9676\n",
      "epoch:  9 step: 1062400, Training Accuracy: 98.44, loss: 35.2336\n",
      "epoch:  9 step: 1075200, Training Accuracy: 98.05, loss: 35.2958\n",
      "epoch:  9 step: 1088000, Training Accuracy: 97.46, loss: 44.4319\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch: 10 step:      0, Training Accuracy: 98.63, loss: 31.4473\n",
      "epoch: 10 step:  12800, Training Accuracy: 97.85, loss: 39.0728\n",
      "epoch: 10 step:  25600, Training Accuracy: 98.24, loss: 32.2492\n",
      "epoch: 10 step:  38400, Training Accuracy: 98.63, loss: 30.2998\n",
      "epoch: 10 step:  51200, Training Accuracy: 97.46, loss: 42.5172\n",
      "epoch: 10 step:  64000, Training Accuracy: 97.66, loss: 28.9720\n",
      "epoch: 10 step:  76800, Training Accuracy: 97.46, loss: 50.2126\n",
      "epoch: 10 step:  89600, Training Accuracy: 97.85, loss: 39.6796\n",
      "epoch: 10 step: 102400, Training Accuracy: 98.83, loss: 30.0730\n",
      "epoch: 10 step: 115200, Training Accuracy: 98.05, loss: 42.0292\n",
      "epoch: 10 step: 128000, Training Accuracy: 99.02, loss: 23.2315\n",
      "epoch: 10 step: 140800, Training Accuracy: 97.85, loss: 37.8181\n",
      "epoch: 10 step: 153600, Training Accuracy: 97.46, loss: 47.3731\n",
      "epoch: 10 step: 166400, Training Accuracy: 98.44, loss: 23.4609\n",
      "epoch: 10 step: 179200, Training Accuracy: 97.66, loss: 45.3320\n",
      "epoch: 10 step: 192000, Training Accuracy: 97.27, loss: 50.1106\n",
      "epoch: 10 step: 204800, Training Accuracy: 97.85, loss: 42.7319\n",
      "epoch: 10 step: 217600, Training Accuracy: 98.24, loss: 29.6765\n",
      "epoch: 10 step: 230400, Training Accuracy: 97.66, loss: 48.6866\n",
      "epoch: 10 step: 243200, Training Accuracy: 98.63, loss: 24.9460\n",
      "epoch: 10 step: 256000, Training Accuracy: 98.63, loss: 28.3312\n",
      "epoch: 10 step: 268800, Training Accuracy: 97.66, loss: 38.4170\n",
      "epoch: 10 step: 281600, Training Accuracy: 97.66, loss: 39.0237\n",
      "epoch: 10 step: 294400, Training Accuracy: 98.05, loss: 42.3196\n",
      "epoch: 10 step: 307200, Training Accuracy: 96.88, loss: 60.1568\n",
      "epoch: 10 step: 320000, Training Accuracy: 97.85, loss: 45.8922\n",
      "epoch: 10 step: 332800, Training Accuracy: 98.44, loss: 30.0434\n",
      "epoch: 10 step: 345600, Training Accuracy: 97.07, loss: 54.8025\n",
      "epoch: 10 step: 358400, Training Accuracy: 98.24, loss: 31.0009\n",
      "epoch: 10 step: 371200, Training Accuracy: 97.85, loss: 37.0853\n",
      "epoch: 10 step: 384000, Training Accuracy: 97.46, loss: 43.1134\n",
      "epoch: 10 step: 396800, Training Accuracy: 98.63, loss: 29.0852\n",
      "epoch: 10 step: 409600, Training Accuracy: 97.07, loss: 47.7512\n",
      "epoch: 10 step: 422400, Training Accuracy: 97.07, loss: 47.5573\n",
      "epoch: 10 step: 435200, Training Accuracy: 97.27, loss: 51.2375\n",
      "epoch: 10 step: 448000, Training Accuracy: 98.05, loss: 33.1855\n",
      "epoch: 10 step: 460800, Training Accuracy: 98.63, loss: 31.7065\n",
      "epoch: 10 step: 473600, Training Accuracy: 98.44, loss: 34.2592\n",
      "epoch: 10 step: 486400, Training Accuracy: 97.46, loss: 48.3506\n",
      "epoch: 10 step: 499200, Training Accuracy: 98.05, loss: 39.4199\n",
      "epoch: 10 step: 512000, Training Accuracy: 97.27, loss: 45.7068\n",
      "epoch: 10 step: 524800, Training Accuracy: 98.24, loss: 28.4013\n",
      "epoch: 10 step: 537600, Training Accuracy: 97.46, loss: 44.2883\n",
      "epoch: 10 step: 550400, Training Accuracy: 97.46, loss: 51.5850\n",
      "epoch: 10 step: 563200, Training Accuracy: 98.05, loss: 30.2679\n",
      "epoch: 10 step: 576000, Training Accuracy: 96.68, loss: 47.0230\n",
      "epoch: 10 step: 588800, Training Accuracy: 98.24, loss: 30.3024\n",
      "epoch: 10 step: 601600, Training Accuracy: 98.83, loss: 24.2366\n",
      "epoch: 10 step: 614400, Training Accuracy: 97.27, loss: 52.4406\n",
      "epoch: 10 step: 627200, Training Accuracy: 98.24, loss: 34.4811\n",
      "epoch: 10 step: 640000, Training Accuracy: 97.66, loss: 43.2444\n",
      "epoch: 10 step: 652800, Training Accuracy: 97.85, loss: 45.4727\n",
      "epoch: 10 step: 665600, Training Accuracy: 98.63, loss: 32.6067\n",
      "epoch: 10 step: 678400, Training Accuracy: 98.24, loss: 33.0709\n",
      "epoch: 10 step: 691200, Training Accuracy: 98.63, loss: 28.7587\n",
      "epoch: 10 step: 704000, Training Accuracy: 97.66, loss: 42.9282\n",
      "epoch: 10 step: 716800, Training Accuracy: 98.83, loss: 32.4295\n",
      "epoch: 10 step: 729600, Training Accuracy: 97.66, loss: 42.7474\n",
      "epoch: 10 step: 742400, Training Accuracy: 97.85, loss: 39.0852\n",
      "epoch: 10 step: 755200, Training Accuracy: 97.27, loss: 48.3963\n",
      "epoch: 10 step: 768000, Training Accuracy: 97.66, loss: 44.7608\n",
      "epoch: 10 step: 780800, Training Accuracy: 98.44, loss: 28.6714\n",
      "epoch: 10 step: 793600, Training Accuracy: 97.66, loss: 35.6650\n",
      "epoch: 10 step: 806400, Training Accuracy: 97.46, loss: 34.8004\n",
      "epoch: 10 step: 819200, Training Accuracy: 98.44, loss: 25.3580\n",
      "epoch: 10 step: 832000, Training Accuracy: 98.05, loss: 33.2626\n",
      "epoch: 10 step: 844800, Training Accuracy: 98.44, loss: 33.6084\n",
      "epoch: 10 step: 857600, Training Accuracy: 98.05, loss: 34.8138\n",
      "epoch: 10 step: 870400, Training Accuracy: 98.44, loss: 34.6252\n",
      "epoch: 10 step: 883200, Training Accuracy: 98.83, loss: 25.3272\n",
      "epoch: 10 step: 896000, Training Accuracy: 97.66, loss: 39.3668\n",
      "epoch: 10 step: 908800, Training Accuracy: 98.24, loss: 30.5555\n",
      "epoch: 10 step: 921600, Training Accuracy: 97.85, loss: 42.2124\n",
      "epoch: 10 step: 934400, Training Accuracy: 97.66, loss: 43.2216\n",
      "epoch: 10 step: 947200, Training Accuracy: 97.07, loss: 54.3278\n",
      "epoch: 10 step: 960000, Training Accuracy: 99.22, loss: 24.6937\n",
      "epoch: 10 step: 972800, Training Accuracy: 99.22, loss: 23.2086\n",
      "epoch: 10 step: 985600, Training Accuracy: 97.27, loss: 47.4537\n",
      "epoch: 10 step: 998400, Training Accuracy: 98.63, loss: 34.5786\n",
      "epoch: 10 step: 1011200, Training Accuracy: 98.83, loss: 28.8475\n",
      "epoch: 10 step: 1024000, Training Accuracy: 97.66, loss: 41.8469\n",
      "epoch: 10 step: 1036800, Training Accuracy: 98.63, loss: 27.9622\n",
      "epoch: 10 step: 1049600, Training Accuracy: 97.27, loss: 41.7585\n",
      "epoch: 10 step: 1062400, Training Accuracy: 96.88, loss: 46.0681\n",
      "epoch: 10 step: 1075200, Training Accuracy: 97.85, loss: 46.0387\n",
      "epoch: 10 step: 1088000, Training Accuracy: 96.68, loss: 39.2713\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n",
      "epoch: 11 step:      0, Training Accuracy: 97.85, loss: 30.5852\n",
      "epoch: 11 step:  12800, Training Accuracy: 98.05, loss: 30.6279\n",
      "epoch: 11 step:  25600, Training Accuracy: 98.63, loss: 24.6883\n",
      "epoch: 11 step:  38400, Training Accuracy: 97.27, loss: 46.6233\n",
      "epoch: 11 step:  51200, Training Accuracy: 97.66, loss: 51.4812\n",
      "epoch: 11 step:  64000, Training Accuracy: 98.83, loss: 20.6715\n",
      "epoch: 11 step:  76800, Training Accuracy: 98.05, loss: 39.2199\n",
      "epoch: 11 step:  89600, Training Accuracy: 97.27, loss: 42.4836\n",
      "epoch: 11 step: 102400, Training Accuracy: 98.24, loss: 35.8961\n",
      "epoch: 11 step: 115200, Training Accuracy: 97.27, loss: 44.0982\n",
      "epoch: 11 step: 128000, Training Accuracy: 98.05, loss: 44.4044\n",
      "epoch: 11 step: 140800, Training Accuracy: 97.66, loss: 48.8929\n",
      "epoch: 11 step: 153600, Training Accuracy: 98.05, loss: 33.4607\n",
      "epoch: 11 step: 166400, Training Accuracy: 97.85, loss: 47.4409\n",
      "epoch: 11 step: 179200, Training Accuracy: 98.63, loss: 29.2287\n",
      "epoch: 11 step: 192000, Training Accuracy: 98.24, loss: 29.6327\n",
      "epoch: 11 step: 204800, Training Accuracy: 96.68, loss: 44.0536\n",
      "epoch: 11 step: 217600, Training Accuracy: 97.46, loss: 43.3042\n",
      "epoch: 11 step: 230400, Training Accuracy: 98.24, loss: 35.8470\n",
      "epoch: 11 step: 243200, Training Accuracy: 99.02, loss: 25.1323\n",
      "epoch: 11 step: 256000, Training Accuracy: 98.83, loss: 27.6882\n",
      "epoch: 11 step: 268800, Training Accuracy: 98.83, loss: 25.7662\n",
      "epoch: 11 step: 281600, Training Accuracy: 99.02, loss: 22.4898\n",
      "epoch: 11 step: 294400, Training Accuracy: 98.44, loss: 32.3351\n",
      "epoch: 11 step: 307200, Training Accuracy: 99.22, loss: 22.7278\n",
      "epoch: 11 step: 320000, Training Accuracy: 97.46, loss: 48.8143\n",
      "epoch: 11 step: 332800, Training Accuracy: 98.24, loss: 37.7495\n",
      "epoch: 11 step: 345600, Training Accuracy: 97.27, loss: 44.6028\n",
      "epoch: 11 step: 358400, Training Accuracy: 97.66, loss: 40.5004\n",
      "epoch: 11 step: 371200, Training Accuracy: 98.24, loss: 27.2165\n",
      "epoch: 11 step: 384000, Training Accuracy: 98.24, loss: 31.7286\n",
      "epoch: 11 step: 396800, Training Accuracy: 98.44, loss: 30.2483\n",
      "epoch: 11 step: 409600, Training Accuracy: 98.24, loss: 32.2240\n",
      "epoch: 11 step: 422400, Training Accuracy: 98.63, loss: 22.0567\n",
      "epoch: 11 step: 435200, Training Accuracy: 98.24, loss: 35.4542\n",
      "epoch: 11 step: 448000, Training Accuracy: 99.02, loss: 27.7116\n",
      "epoch: 11 step: 460800, Training Accuracy: 98.63, loss: 36.9034\n",
      "epoch: 11 step: 473600, Training Accuracy: 96.88, loss: 44.3381\n",
      "epoch: 11 step: 486400, Training Accuracy: 97.66, loss: 36.5245\n",
      "epoch: 11 step: 499200, Training Accuracy: 97.27, loss: 44.3057\n",
      "epoch: 11 step: 512000, Training Accuracy: 99.02, loss: 24.7790\n",
      "epoch: 11 step: 524800, Training Accuracy: 98.24, loss: 34.3260\n",
      "epoch: 11 step: 537600, Training Accuracy: 96.68, loss: 60.8613\n",
      "epoch: 11 step: 550400, Training Accuracy: 98.63, loss: 26.4649\n",
      "epoch: 11 step: 563200, Training Accuracy: 98.05, loss: 38.5064\n",
      "epoch: 11 step: 576000, Training Accuracy: 98.24, loss: 34.0521\n",
      "epoch: 11 step: 588800, Training Accuracy: 98.05, loss: 37.4246\n",
      "epoch: 11 step: 601600, Training Accuracy: 98.24, loss: 41.1434\n",
      "epoch: 11 step: 614400, Training Accuracy: 98.63, loss: 28.0040\n",
      "epoch: 11 step: 627200, Training Accuracy: 97.07, loss: 54.7867\n",
      "epoch: 11 step: 640000, Training Accuracy: 99.22, loss: 28.9259\n",
      "epoch: 11 step: 652800, Training Accuracy: 97.46, loss: 42.6587\n",
      "epoch: 11 step: 665600, Training Accuracy: 97.66, loss: 47.1719\n",
      "epoch: 11 step: 678400, Training Accuracy: 97.27, loss: 53.0009\n",
      "epoch: 11 step: 691200, Training Accuracy: 98.83, loss: 30.9839\n",
      "epoch: 11 step: 704000, Training Accuracy: 97.46, loss: 45.5657\n",
      "epoch: 11 step: 716800, Training Accuracy: 99.02, loss: 26.2406\n",
      "epoch: 11 step: 729600, Training Accuracy: 98.63, loss: 25.0097\n",
      "epoch: 11 step: 742400, Training Accuracy: 97.66, loss: 43.5181\n",
      "epoch: 11 step: 755200, Training Accuracy: 96.88, loss: 55.2397\n",
      "epoch: 11 step: 768000, Training Accuracy: 97.85, loss: 42.2758\n",
      "epoch: 11 step: 780800, Training Accuracy: 97.27, loss: 52.8364\n",
      "epoch: 11 step: 793600, Training Accuracy: 98.63, loss: 29.1520\n",
      "epoch: 11 step: 806400, Training Accuracy: 97.85, loss: 35.1076\n",
      "epoch: 11 step: 819200, Training Accuracy: 98.63, loss: 22.4345\n",
      "epoch: 11 step: 832000, Training Accuracy: 98.24, loss: 32.3623\n",
      "epoch: 11 step: 844800, Training Accuracy: 97.85, loss: 42.2141\n",
      "epoch: 11 step: 857600, Training Accuracy: 97.85, loss: 38.9883\n",
      "epoch: 11 step: 870400, Training Accuracy: 97.66, loss: 46.3612\n",
      "epoch: 11 step: 883200, Training Accuracy: 98.83, loss: 26.5467\n",
      "epoch: 11 step: 896000, Training Accuracy: 99.22, loss: 24.2738\n",
      "epoch: 11 step: 908800, Training Accuracy: 97.66, loss: 39.1677\n",
      "epoch: 11 step: 921600, Training Accuracy: 98.63, loss: 36.7975\n",
      "epoch: 11 step: 934400, Training Accuracy: 98.44, loss: 31.3061\n",
      "epoch: 11 step: 947200, Training Accuracy: 98.05, loss: 40.8064\n",
      "epoch: 11 step: 960000, Training Accuracy: 97.27, loss: 41.2463\n",
      "epoch: 11 step: 972800, Training Accuracy: 99.22, loss: 20.1818\n",
      "epoch: 11 step: 985600, Training Accuracy: 98.44, loss: 34.1938\n",
      "epoch: 11 step: 998400, Training Accuracy: 97.27, loss: 46.0204\n",
      "epoch: 11 step: 1011200, Training Accuracy: 98.24, loss: 35.3947\n",
      "epoch: 11 step: 1024000, Training Accuracy: 97.66, loss: 36.6739\n",
      "epoch: 11 step: 1036800, Training Accuracy: 96.29, loss: 59.9130\n",
      "epoch: 11 step: 1049600, Training Accuracy: 98.05, loss: 33.2217\n",
      "epoch: 11 step: 1062400, Training Accuracy: 97.66, loss: 46.9731\n",
      "epoch: 11 step: 1075200, Training Accuracy: 97.27, loss: 56.4713\n",
      "epoch: 11 step: 1088000, Training Accuracy: 97.85, loss: 35.1326\n",
      "**** SAVED MODEL ****\n",
      "**** COMPLETED EPOCH ****\n"
     ]
    }
   ],
   "source": [
    "checkpoint_dir = \"full_model/\"\n",
    "tf.gfile.MakeDirs(checkpoint_dir)\n",
    "\n",
    "learningRates = np.hstack((1e-3*np.ones(6),\n",
    "                           1e-4*np.ones(4),\n",
    "                           1e-5*np.ones(2)))\n",
    "nEpochs = len(learningRates)\n",
    "trainBatchSize = 512\n",
    "nDataSamples = len(data)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver = tf.train.Saver()\n",
    "    tf.train.write_graph(sess.graph_def,\n",
    "                         checkpoint_dir,\n",
    "                         \"graph.pbtxt\",\n",
    "                         True)\n",
    "    sess.run(tf.global_variables_initializer())    \n",
    "    \n",
    "    for epoch in np.arange(nEpochs):\n",
    "        idx = np.arange(len(labels))\n",
    "        np.random.shuffle(idx)\n",
    "        \n",
    "        for i in np.arange(0, nDataSamples, trainBatchSize):\n",
    "            x_batch = data[idx[i:i+trainBatchSize], :, :]\n",
    "            y_batch = labels[idx[i:i+trainBatchSize]]\n",
    "            \n",
    "            feed = {x: x_batch,\n",
    "                    Y_: y_batch,\n",
    "                   learning_rate: learningRates[epoch],\n",
    "                   keep_prob: 0.75}\n",
    "            sess.run(train_op, feed_dict=feed)\n",
    "            \n",
    "            if i%50*trainBatchSize == 0:\n",
    "                feed = {x: x_batch,\n",
    "                        Y_: y_batch,\n",
    "                        learning_rate: learningRates[epoch],\n",
    "                        keep_prob: 1.0}\n",
    "                \n",
    "                train_accuracy, loss_value = sess.run([accuracy, cross_entropy], feed_dict=feed)\n",
    "                print(\"epoch: %2d step: %6d, Training Accuracy: %3.2f, loss: %6.4f\" % \\\n",
    "                      (epoch, i, train_accuracy*100, loss_value))\n",
    "                \n",
    "        tf.gfile.MakeDirs(checkpoint_dir + '/model' + str(epoch))\n",
    "        checkpoint_file = os.path.join(checkpoint_dir + '/model' + str(epoch), \"model\")\n",
    "        saver.save(sess, checkpoint_file)\n",
    "        print(\"**** SAVED MODEL ****\")      \n",
    "        print(\"**** COMPLETED EPOCH ****\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
